diff --git a/.golangci.yml b/.golangci.yml index 4950b98c21baf..b4aa0a01d7795 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -19,10 +19,24 @@ linters: - govet - ineffassign - misspell - # - staticcheck - unconvert - # - unused - varcheck + - typecheck + - unused + - staticcheck + - bidichk + - durationcheck + - exportloopref + - gosec + + #- structcheck # lots of false positives + #- errcheck #lot of false positives + # - contextcheck + # - errchkjson # lots of false positives + # - errorlint # this check crashes + # - exhaustive # silly check + # - makezero # false positives + # - nilerr # several intentional linters-settings: gofmt: @@ -30,21 +44,28 @@ linters-settings: goconst: min-len: 3 # minimum length of string constant min-occurrences: 6 # minimum number of occurrences + gosec: + excludes: + - G404 # Use of weak random number generator - lots of FP + - G107 # Potential http request -- those are intentional + - G306 # G306: Expect WriteFile permissions to be 0600 or less issues: exclude-rules: - - path: crypto/blake2b/ - linters: - - deadcode - - path: crypto/bn256/cloudflare - linters: - - deadcode - - path: p2p/discv5/ - linters: - - deadcode - - path: core/vm/instructions_test.go - linters: - - goconst - - path: cmd/faucet/ + - path: crypto/bn256/cloudflare/optate.go linters: - deadcode + - staticcheck + - path: internal/build/pgp.go + text: 'SA1019: package golang.org/x/crypto/openpgp is deprecated' + - path: core/vm/contracts.go + text: 'SA1019: package golang.org/x/crypto/ripemd160 is deprecated' + - path: accounts/usbwallet/trezor.go + text: 'SA1019: package github.com/golang/protobuf/proto is deprecated' + - path: accounts/usbwallet/trezor/ + text: 'SA1019: package github.com/golang/protobuf/proto is deprecated' + exclude: + - 'SA1019: event.TypeMux is deprecated: use Feed' + - 'SA1019: strings.Title is deprecated' + - 'SA1029: should not use built-in type string as key for value' + - 'G306: Expect WriteFile permissions to be 0600 or less' \ No newline at end of file diff --git a/README.md b/README.md index e3c1b1550975e..76119c7a6fbdc 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ directory. | `abigen` | Source code generator to convert Ethereum contract definitions into easy to use, compile-time type-safe Go packages. It operates on plain [Ethereum contract ABIs](https://docs.soliditylang.org/en/develop/abi-spec.html) with expanded functionality if the contract bytecode is also available. However, it also accepts Solidity source files, making development much more streamlined. Please see our [Native DApps](https://geth.ethereum.org/docs/dapp/native-bindings) page for details. | | `bootnode` | Stripped down version of our Ethereum client implementation that only takes part in the network node discovery protocol, but does not run any of the higher level application protocols. It can be used as a lightweight bootstrap node to aid in finding peers in private networks. | | `evm` | Developer utility version of the EVM (Ethereum Virtual Machine) that is capable of running bytecode snippets within a configurable environment and execution mode. Its purpose is to allow isolated, fine-grained debugging of EVM opcodes (e.g. `evm --code 60ff60ff --debug run`). | -| `rlpdump` | Developer utility tool to convert binary RLP ([Recursive Length Prefix](https://eth.wiki/en/fundamentals/rlp)) dumps (data encoding used by the Ethereum protocol both network as well as consensus wise) to user-friendlier hierarchical representation (e.g. `rlpdump --hex CE0183FFFFFFC4C304050583616263`). | +| `rlpdump` | Developer utility tool to convert binary RLP ([Recursive Length Prefix](https://ethereum.org/en/developers/docs/data-structures-and-encoding/rlp)) dumps (data encoding used by the Ethereum protocol both network as well as consensus wise) to user-friendlier hierarchical representation (e.g. `rlpdump --hex CE0183FFFFFFC4C304050583616263`). | | `puppeth` | a CLI wizard that aids in creating a new Ethereum network. | ## Running `geth` @@ -202,7 +202,7 @@ accessible from the outside. As a developer, sooner rather than later you'll want to start interacting with `geth` and the Ethereum network via your own programs and not manually through the console. To aid -this, `geth` has built-in support for a JSON-RPC based APIs ([standard APIs](https://eth.wiki/json-rpc/API) +this, `geth` has built-in support for a JSON-RPC based APIs ([standard APIs](https://ethereum.github.io/execution-apis/api-documentation/) and [`geth` specific APIs](https://geth.ethereum.org/docs/rpc/server)). These can be exposed via HTTP, WebSockets and IPC (UNIX sockets on UNIX based platforms, and named pipes on Windows). @@ -311,7 +311,7 @@ $ bootnode --genkey=boot.key $ bootnode --nodekey=boot.key ``` -With the bootnode online, it will display an [`enode` URL](https://eth.wiki/en/fundamentals/enode-url-format) +With the bootnode online, it will display an [`enode` URL](https://ethereum.org/en/developers/docs/networking-layer/network-addresses/#enode) that other nodes can use to connect to it and exchange peer information. Make sure to replace the displayed IP address information (most probably `[::]`) with your externally accessible IP to get the actual `enode` URL. diff --git a/accounts/abi/abi.go b/accounts/abi/abi.go index cd2f4d7978bd1..ed5b6e92ef9c3 100644 --- a/accounts/abi/abi.go +++ b/accounts/abi/abi.go @@ -164,7 +164,7 @@ func (abi *ABI) UnmarshalJSON(data []byte) error { case "constructor": abi.Constructor = NewMethod("", "", Constructor, field.StateMutability, field.Constant, field.Payable, field.Inputs, nil) case "function": - name := overloadedName(field.Name, func(s string) bool { _, ok := abi.Methods[s]; return ok }) + name := ResolveNameConflict(field.Name, func(s string) bool { _, ok := abi.Methods[s]; return ok }) abi.Methods[name] = NewMethod(name, field.Name, Function, field.StateMutability, field.Constant, field.Payable, field.Inputs, field.Outputs) case "fallback": // New introduced function type in v0.6.0, check more detail @@ -184,9 +184,11 @@ func (abi *ABI) UnmarshalJSON(data []byte) error { } abi.Receive = NewMethod("", "", Receive, field.StateMutability, field.Constant, field.Payable, nil, nil) case "event": - name := overloadedName(field.Name, func(s string) bool { _, ok := abi.Events[s]; return ok }) + name := ResolveNameConflict(field.Name, func(s string) bool { _, ok := abi.Events[s]; return ok }) abi.Events[name] = NewEvent(name, field.Name, field.Anonymous, field.Inputs) case "error": + // Errors cannot be overloaded or overridden but are inherited, + // no need to resolve the name conflict here. abi.Errors[field.Name] = NewError(field.Name, field.Inputs) default: return fmt.Errorf("abi: could not recognize type %v of field %v", field.Type, field.Name) @@ -251,20 +253,3 @@ func UnpackRevert(data []byte) (string, error) { } return unpacked[0].(string), nil } - -// overloadedName returns the next available name for a given thing. -// Needed since solidity allows for overloading. -// -// e.g. if the abi contains Methods send, send1 -// overloadedName would return send2 for input send. -// -// overloadedName works for methods, events and errors. -func overloadedName(rawName string, isAvail func(string) bool) string { - name := rawName - ok := isAvail(name) - for idx := 0; ok; idx++ { - name = fmt.Sprintf("%s%d", rawName, idx) - ok = isAvail(name) - } - return name -} diff --git a/accounts/abi/abi_test.go b/accounts/abi/abi_test.go index cc8dfc61c389f..cd9433ca7f73a 100644 --- a/accounts/abi/abi_test.go +++ b/accounts/abi/abi_test.go @@ -1038,9 +1038,7 @@ func TestABI_EventById(t *testing.T) { } if event == nil { t.Errorf("We should find a event for topic %s, test #%d", topicID.Hex(), testnum) - } - - if event.ID != topicID { + } else if event.ID != topicID { t.Errorf("Event id %s does not match topic %s, test #%d", event.ID.Hex(), topicID.Hex(), testnum) } diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go index ac696f446be6a..f5780c4a32c27 100644 --- a/accounts/abi/bind/backends/simulated.go +++ b/accounts/abi/bind/backends/simulated.go @@ -63,9 +63,10 @@ type SimulatedBackend struct { database ethdb.Database // In memory database to store our testing data blockchain *core.BlockChain // Ethereum blockchain to handle the consensus - mu sync.Mutex - pendingBlock *types.Block // Currently pending block that will be imported on request - pendingState *state.StateDB // Currently pending state that will be the active on request + mu sync.Mutex + pendingBlock *types.Block // Currently pending block that will be imported on request + pendingState *state.StateDB // Currently pending state that will be the active on request + pendingReceipts types.Receipts // Currently receipts for the pending block events *filters.EventSystem // Event system for filtering log events live @@ -84,8 +85,8 @@ func NewSimulatedBackendWithDatabase(database ethdb.Database, alloc core.Genesis database: database, blockchain: blockchain, config: genesis.Config, - events: filters.NewEventSystem(&filterBackend{database, blockchain}, false), } + backend.events = filters.NewEventSystem(&filterBackend{database, blockchain, backend}, false) backend.rollback(blockchain.CurrentBlock()) return backend } @@ -662,7 +663,7 @@ func (b *SimulatedBackend) SendTransaction(ctx context.Context, tx *types.Transa return fmt.Errorf("invalid transaction nonce: got %d, want %d", tx.Nonce(), nonce) } // Include tx in chain - blocks, _ := core.GenerateChain(b.config, block, ethash.NewFaker(), b.database, 1, func(number int, block *core.BlockGen) { + blocks, receipts := core.GenerateChain(b.config, block, ethash.NewFaker(), b.database, 1, func(number int, block *core.BlockGen) { for _, tx := range b.pendingBlock.Transactions() { block.AddTxWithChain(b.blockchain, tx) } @@ -672,6 +673,7 @@ func (b *SimulatedBackend) SendTransaction(ctx context.Context, tx *types.Transa b.pendingBlock = blocks[0] b.pendingState, _ = state.New(b.pendingBlock.Root(), stateDB.Database(), nil) + b.pendingReceipts = receipts[0] return nil } @@ -683,7 +685,7 @@ func (b *SimulatedBackend) FilterLogs(ctx context.Context, query ethereum.Filter var filter *filters.Filter if query.BlockHash != nil { // Block filter requested, construct a single-shot filter - filter = filters.NewBlockFilter(&filterBackend{b.database, b.blockchain}, *query.BlockHash, query.Addresses, query.Topics) + filter = filters.NewBlockFilter(&filterBackend{b.database, b.blockchain, b}, *query.BlockHash, query.Addresses, query.Topics) } else { // Initialize unset filter boundaries to run from genesis to chain head from := int64(0) @@ -695,7 +697,7 @@ func (b *SimulatedBackend) FilterLogs(ctx context.Context, query ethereum.Filter to = query.ToBlock.Int64() } // Construct the range filter - filter = filters.NewRangeFilter(&filterBackend{b.database, b.blockchain}, from, to, query.Addresses, query.Topics) + filter = filters.NewRangeFilter(&filterBackend{b.database, b.blockchain, b}, from, to, query.Addresses, query.Topics) } // Run the filter and return all the logs logs, err := filter.Logs(ctx) @@ -816,8 +818,9 @@ func (m callMsg) AccessList() types.AccessList { return m.CallMsg.AccessList } // filterBackend implements filters.Backend to support filtering for logs without // taking bloom-bits acceleration structures into account. type filterBackend struct { - db ethdb.Database - bc *core.BlockChain + db ethdb.Database + bc *core.BlockChain + backend *SimulatedBackend } func (fb *filterBackend) ChainDb() ethdb.Database { return fb.db } @@ -834,6 +837,10 @@ func (fb *filterBackend) HeaderByHash(ctx context.Context, hash common.Hash) (*t return fb.bc.GetHeaderByHash(hash), nil } +func (fb *filterBackend) PendingBlockAndReceipts() (*types.Block, types.Receipts) { + return fb.backend.pendingBlock, fb.backend.pendingReceipts +} + func (fb *filterBackend) GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) { number := rawdb.ReadHeaderNumber(fb.db, hash) if number == nil { diff --git a/accounts/abi/bind/backends/simulated_test.go b/accounts/abi/bind/backends/simulated_test.go index 8a0cbe335778b..2287055ebdfb5 100644 --- a/accounts/abi/bind/backends/simulated_test.go +++ b/accounts/abi/bind/backends/simulated_test.go @@ -655,8 +655,7 @@ func TestHeaderByNumber(t *testing.T) { } if latestBlockHeader == nil { t.Errorf("received a nil block header") - } - if latestBlockHeader.Number.Uint64() != uint64(0) { + } else if latestBlockHeader.Number.Uint64() != uint64(0) { t.Errorf("expected block header number 0, instead got %v", latestBlockHeader.Number.Uint64()) } diff --git a/accounts/abi/bind/bind.go b/accounts/abi/bind/bind.go index 2bd8b6dde07f0..a938e7dfcd85a 100644 --- a/accounts/abi/bind/bind.go +++ b/accounts/abi/bind/bind.go @@ -99,6 +99,7 @@ func Bind(types []string, abis []string, bytecodes []string, fsigs []map[string] // Normalize the method for capital cases and non-anonymous inputs/outputs normalized := original normalizedName := methodNormalizer[lang](alias(aliases, original.Name)) + // Ensure there is no duplicated identifier var identifiers = callIdentifiers if !original.IsConstant() { @@ -108,6 +109,7 @@ func Bind(types []string, abis []string, bytecodes []string, fsigs []map[string] return "", fmt.Errorf("duplicated identifier \"%s\"(normalized \"%s\"), use --alias for renaming", original.Name, normalizedName) } identifiers[normalizedName] = true + normalized.Name = normalizedName normalized.Inputs = make([]abi.Argument, len(original.Inputs)) copy(normalized.Inputs, original.Inputs) @@ -152,12 +154,22 @@ func Bind(types []string, abis []string, bytecodes []string, fsigs []map[string] eventIdentifiers[normalizedName] = true normalized.Name = normalizedName + used := make(map[string]bool) normalized.Inputs = make([]abi.Argument, len(original.Inputs)) copy(normalized.Inputs, original.Inputs) for j, input := range normalized.Inputs { if input.Name == "" { normalized.Inputs[j].Name = fmt.Sprintf("arg%d", j) } + // Event is a bit special, we need to define event struct in binding, + // ensure there is no camel-case-style name conflict. + for index := 0; ; index++ { + if !used[capitalise(normalized.Inputs[j].Name)] { + used[capitalise(normalized.Inputs[j].Name)] = true + break + } + normalized.Inputs[j].Name = fmt.Sprintf("%s%d", normalized.Inputs[j].Name, index) + } if hasStruct(input.Type) { bindStructType[lang](input.Type, structs) } @@ -432,15 +444,22 @@ func bindStructTypeGo(kind abi.Type, structs map[string]*tmplStruct) string { if s, exist := structs[id]; exist { return s.Name } - var fields []*tmplField + var ( + names = make(map[string]bool) + fields []*tmplField + ) for i, elem := range kind.TupleElems { - field := bindStructTypeGo(*elem, structs) - fields = append(fields, &tmplField{Type: field, Name: capitalise(kind.TupleRawNames[i]), SolKind: *elem}) + name := capitalise(kind.TupleRawNames[i]) + name = abi.ResolveNameConflict(name, func(s string) bool { return names[s] }) + names[name] = true + fields = append(fields, &tmplField{Type: bindStructTypeGo(*elem, structs), Name: name, SolKind: *elem}) } name := kind.TupleRawName if name == "" { name = fmt.Sprintf("Struct%d", len(structs)) } + name = capitalise(name) + structs[id] = &tmplStruct{ Name: name, Fields: fields, diff --git a/accounts/abi/bind/bind_test.go b/accounts/abi/bind/bind_test.go index 87e8187f09ca0..b71d85ee6085d 100644 --- a/accounts/abi/bind/bind_test.go +++ b/accounts/abi/bind/bind_test.go @@ -1948,6 +1948,54 @@ var bindTests = []struct { } sim.Commit() + if _, err = bind.WaitDeployed(nil, sim, tx); err != nil { + t.Logf("Deployment tx: %+v", tx) + t.Errorf("bind.WaitDeployed(nil, %T, ) got err %v; want nil err", sim, err) + } + `, + }, + { + name: `NameConflict`, + contract: ` + // SPDX-License-Identifier: GPL-3.0 + pragma solidity >=0.4.22 <0.9.0; + contract oracle { + struct request { + bytes data; + bytes _data; + } + event log (int msg, int _msg); + function addRequest(request memory req) public pure {} + function getRequest() pure public returns (request memory) { + return request("", ""); + } + } + `, + bytecode: []string{`0x608060405234801561001057600080fd5b5061042b806100206000396000f3fe608060405234801561001057600080fd5b50600436106100365760003560e01c8063c2bb515f1461003b578063cce7b04814610059575b600080fd5b610043610075565b60405161005091906101af565b60405180910390f35b610073600480360381019061006e91906103ac565b6100b5565b005b61007d6100b8565b604051806040016040528060405180602001604052806000815250815260200160405180602001604052806000815250815250905090565b50565b604051806040016040528060608152602001606081525090565b600081519050919050565b600082825260208201905092915050565b60005b8381101561010c5780820151818401526020810190506100f1565b8381111561011b576000848401525b50505050565b6000601f19601f8301169050919050565b600061013d826100d2565b61014781856100dd565b93506101578185602086016100ee565b61016081610121565b840191505092915050565b600060408301600083015184820360008601526101888282610132565b915050602083015184820360208601526101a28282610132565b9150508091505092915050565b600060208201905081810360008301526101c9818461016b565b905092915050565b6000604051905090565b600080fd5b600080fd5b600080fd5b7f4e487b7100000000000000000000000000000000000000000000000000000000600052604160045260246000fd5b61022282610121565b810181811067ffffffffffffffff82111715610241576102406101ea565b5b80604052505050565b60006102546101d1565b90506102608282610219565b919050565b600080fd5b600080fd5b600080fd5b600067ffffffffffffffff82111561028f5761028e6101ea565b5b61029882610121565b9050602081019050919050565b82818337600083830152505050565b60006102c76102c284610274565b61024a565b9050828152602081018484840111156102e3576102e261026f565b5b6102ee8482856102a5565b509392505050565b600082601f83011261030b5761030a61026a565b5b813561031b8482602086016102b4565b91505092915050565b60006040828403121561033a576103396101e5565b5b610344604061024a565b9050600082013567ffffffffffffffff81111561036457610363610265565b5b610370848285016102f6565b600083015250602082013567ffffffffffffffff81111561039457610393610265565b5b6103a0848285016102f6565b60208301525092915050565b6000602082840312156103c2576103c16101db565b5b600082013567ffffffffffffffff8111156103e0576103df6101e0565b5b6103ec84828501610324565b9150509291505056fea264697066735822122033bca1606af9b6aeba1673f98c52003cec19338539fb44b86690ce82c51483b564736f6c634300080e0033`}, + abi: []string{`[ { "anonymous": false, "inputs": [ { "indexed": false, "internalType": "int256", "name": "msg", "type": "int256" }, { "indexed": false, "internalType": "int256", "name": "_msg", "type": "int256" } ], "name": "log", "type": "event" }, { "inputs": [ { "components": [ { "internalType": "bytes", "name": "data", "type": "bytes" }, { "internalType": "bytes", "name": "_data", "type": "bytes" } ], "internalType": "struct oracle.request", "name": "req", "type": "tuple" } ], "name": "addRequest", "outputs": [], "stateMutability": "pure", "type": "function" }, { "inputs": [], "name": "getRequest", "outputs": [ { "components": [ { "internalType": "bytes", "name": "data", "type": "bytes" }, { "internalType": "bytes", "name": "_data", "type": "bytes" } ], "internalType": "struct oracle.request", "name": "", "type": "tuple" } ], "stateMutability": "pure", "type": "function" } ]`}, + imports: ` + "math/big" + + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/eth/ethconfig" + `, + tester: ` + var ( + key, _ = crypto.GenerateKey() + user, _ = bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) + sim = backends.NewSimulatedBackend(core.GenesisAlloc{user.From: {Balance: big.NewInt(1000000000000000000)}}, ethconfig.Defaults.Miner.GasCeil) + ) + defer sim.Close() + + _, tx, _, err := DeployNameConflict(user, sim) + if err != nil { + t.Fatalf("DeployNameConflict() got err %v; want nil err", err) + } + sim.Commit() + if _, err = bind.WaitDeployed(nil, sim, tx); err != nil { t.Logf("Deployment tx: %+v", tx) t.Errorf("bind.WaitDeployed(nil, %T, ) got err %v; want nil err", sim, err) diff --git a/accounts/abi/error.go b/accounts/abi/error.go index e564c10c2f840..f53c996def14d 100644 --- a/accounts/abi/error.go +++ b/accounts/abi/error.go @@ -30,11 +30,13 @@ type Error struct { Name string Inputs Arguments str string + // Sig contains the string signature according to the ABI spec. - // e.g. event foo(uint32 a, int b) = "foo(uint32,int256)" + // e.g. error foo(uint32 a, int b) = "foo(uint32,int256)" // Please note that "int" is substitute for its canonical representation "int256" Sig string - // ID returns the canonical representation of the event's signature used by the + + // ID returns the canonical representation of the error's signature used by the // abi definition to identify event names and types. ID common.Hash } diff --git a/accounts/abi/event.go b/accounts/abi/event.go index b238a36d7ceab..f9457b86afebf 100644 --- a/accounts/abi/event.go +++ b/accounts/abi/event.go @@ -29,24 +29,27 @@ import ( // don't get the signature canonical representation as the first LOG topic. type Event struct { // Name is the event name used for internal representation. It's derived from - // the raw name and a suffix will be added in the case of a event overload. + // the raw name and a suffix will be added in the case of event overloading. // // e.g. // These are two events that have the same name: // * foo(int,int) // * foo(uint,uint) - // The event name of the first one wll be resolved as foo while the second one + // The event name of the first one will be resolved as foo while the second one // will be resolved as foo0. Name string + // RawName is the raw event name parsed from ABI. RawName string Anonymous bool Inputs Arguments str string + // Sig contains the string signature according to the ABI spec. // e.g. event foo(uint32 a, int b) = "foo(uint32,int256)" // Please note that "int" is substitute for its canonical representation "int256" Sig string + // ID returns the canonical representation of the event's signature used by the // abi definition to identify event names and types. ID common.Hash diff --git a/accounts/abi/type.go b/accounts/abi/type.go index fd75f586a99db..008b665b1aeeb 100644 --- a/accounts/abi/type.go +++ b/accounts/abi/type.go @@ -163,22 +163,26 @@ func NewType(t string, internalType string, components []ArgumentMarshaling) (ty elems []*Type names []string expression string // canonical parameter expression + used = make(map[string]bool) ) expression += "(" - overloadedNames := make(map[string]string) for idx, c := range components { cType, err := NewType(c.Type, c.InternalType, c.Components) if err != nil { return Type{}, err } - fieldName, err := overloadedArgName(c.Name, overloadedNames) + name := ToCamelCase(c.Name) + if name == "" { + return Type{}, errors.New("abi: purely anonymous or underscored field is not supported") + } + fieldName := ResolveNameConflict(name, func(s string) bool { return used[s] }) if err != nil { return Type{}, err } + used[fieldName] = true if !isValidFieldName(fieldName) { return Type{}, fmt.Errorf("field %d has invalid name", idx) } - overloadedNames[fieldName] = fieldName fields = append(fields, reflect.StructField{ Name: fieldName, // reflect.StructOf will panic for any exported field. Type: cType.GetType(), @@ -255,20 +259,6 @@ func (t Type) GetType() reflect.Type { } } -func overloadedArgName(rawName string, names map[string]string) (string, error) { - fieldName := ToCamelCase(rawName) - if fieldName == "" { - return "", errors.New("abi: purely anonymous or underscored field is not supported") - } - // Handle overloaded fieldNames - _, ok := names[fieldName] - for idx := 0; ok; idx++ { - fieldName = fmt.Sprintf("%s%d", ToCamelCase(rawName), idx) - _, ok = names[fieldName] - } - return fieldName, nil -} - // String implements Stringer. func (t Type) String() (out string) { return t.stringKind diff --git a/accounts/abi/unpack.go b/accounts/abi/unpack.go index 43cd6c64575c7..28c5c82bb3d5e 100644 --- a/accounts/abi/unpack.go +++ b/accounts/abi/unpack.go @@ -255,7 +255,7 @@ func toGoType(index int, t Type, output []byte) (interface{}, error) { // lengthPrefixPointsTo interprets a 32 byte slice as an offset and then determines which indices to look to decode the type. func lengthPrefixPointsTo(index int, output []byte) (start int, length int, err error) { - bigOffsetEnd := big.NewInt(0).SetBytes(output[index : index+32]) + bigOffsetEnd := new(big.Int).SetBytes(output[index : index+32]) bigOffsetEnd.Add(bigOffsetEnd, common.Big32) outputLength := big.NewInt(int64(len(output))) @@ -268,11 +268,9 @@ func lengthPrefixPointsTo(index int, output []byte) (start int, length int, err } offsetEnd := int(bigOffsetEnd.Uint64()) - lengthBig := big.NewInt(0).SetBytes(output[offsetEnd-32 : offsetEnd]) + lengthBig := new(big.Int).SetBytes(output[offsetEnd-32 : offsetEnd]) - totalSize := big.NewInt(0) - totalSize.Add(totalSize, bigOffsetEnd) - totalSize.Add(totalSize, lengthBig) + totalSize := new(big.Int).Add(bigOffsetEnd, lengthBig) if totalSize.BitLen() > 63 { return 0, 0, fmt.Errorf("abi: length larger than int64: %v", totalSize) } @@ -287,7 +285,7 @@ func lengthPrefixPointsTo(index int, output []byte) (start int, length int, err // tuplePointsTo resolves the location reference for dynamic tuple. func tuplePointsTo(index int, output []byte) (start int, err error) { - offset := big.NewInt(0).SetBytes(output[index : index+32]) + offset := new(big.Int).SetBytes(output[index : index+32]) outputLen := big.NewInt(int64(len(output))) if offset.Cmp(outputLen) > 0 { diff --git a/accounts/abi/unpack_test.go b/accounts/abi/unpack_test.go index bf40c301b5f7c..ae3565c71e29e 100644 --- a/accounts/abi/unpack_test.go +++ b/accounts/abi/unpack_test.go @@ -424,7 +424,7 @@ func TestMultiReturnWithStringArray(t *testing.T) { } buff := new(bytes.Buffer) buff.Write(common.Hex2Bytes("000000000000000000000000000000000000000000000000000000005c1b78ea0000000000000000000000000000000000000000000000000000000000000006000000000000000000000000000000000000000000000001a055690d9db80000000000000000000000000000ab1257528b3782fb40d7ed5f72e624b744dffb2f00000000000000000000000000000000000000000000000000000000000000c00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000008457468657265756d000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001048656c6c6f2c20457468657265756d2100000000000000000000000000000000")) - temp, _ := big.NewInt(0).SetString("30000000000000000000", 10) + temp, _ := new(big.Int).SetString("30000000000000000000", 10) ret1, ret1Exp := new([3]*big.Int), [3]*big.Int{big.NewInt(1545304298), big.NewInt(6), temp} ret2, ret2Exp := new(common.Address), common.HexToAddress("ab1257528b3782fb40d7ed5f72e624b744dffb2f") ret3, ret3Exp := new([2]string), [2]string{"Ethereum", "Hello, Ethereum!"} diff --git a/accounts/abi/utils.go b/accounts/abi/utils.go new file mode 100644 index 0000000000000..e24df5b7003d8 --- /dev/null +++ b/accounts/abi/utils.go @@ -0,0 +1,41 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package abi + +import "fmt" + +// ResolveNameConflict returns the next available name for a given thing. +// This helper can be used for lots of purposes: +// +// - In solidity function overloading is supported, this function can fix +// the name conflicts of overloaded functions. +// - In golang binding generation, the parameter(in function, event, error, +// and struct definition) name will be converted to camelcase style which +// may eventually lead to name conflicts. +// +// Name conflicts are mostly resolved by adding number suffix. +// e.g. if the abi contains Methods send, send1 +// ResolveNameConflict would return send2 for input send. +func ResolveNameConflict(rawName string, used func(string) bool) string { + name := rawName + ok := used(name) + for idx := 0; ok; idx++ { + name = fmt.Sprintf("%s%d", rawName, idx) + ok = used(name) + } + return name +} diff --git a/accounts/external/backend.go b/accounts/external/backend.go index e3f754eafcc42..d403b7e562d41 100644 --- a/accounts/external/backend.go +++ b/accounts/external/backend.go @@ -152,10 +152,6 @@ func (api *ExternalSigner) SelfDerive(bases []accounts.DerivationPath, chain eth log.Error("operation SelfDerive not supported on external signers") } -func (api *ExternalSigner) signHash(account accounts.Account, hash []byte) ([]byte, error) { - return []byte{}, fmt.Errorf("operation not supported on external signers") -} - // SignData signs keccak256(data). The mimetype parameter describes the type of data being signed func (api *ExternalSigner) SignData(account accounts.Account, mimeType string, data []byte) ([]byte, error) { var res hexutil.Bytes diff --git a/accounts/keystore/account_cache_test.go b/accounts/keystore/account_cache_test.go index fda0e5667c2aa..bdcd811825128 100644 --- a/accounts/keystore/account_cache_test.go +++ b/accounts/keystore/account_cache_test.go @@ -383,7 +383,7 @@ func TestUpdatedKeyfileContents(t *testing.T) { time.Sleep(1000 * time.Millisecond) // Now replace file contents with crap - if err := os.WriteFile(file, []byte("foo"), 0644); err != nil { + if err := os.WriteFile(file, []byte("foo"), 0600); err != nil { t.Fatal(err) return } diff --git a/accounts/keystore/passphrase_test.go b/accounts/keystore/passphrase_test.go index b94fce8edcae7..1356b317806dc 100644 --- a/accounts/keystore/passphrase_test.go +++ b/accounts/keystore/passphrase_test.go @@ -52,7 +52,7 @@ func TestKeyEncryptDecrypt(t *testing.T) { t.Errorf("test %d: key address mismatch: have %x, want %x", i, key.Address, address) } // Recrypt with a new password and start over - password += "new data appended" + password += "new data appended" // nolint: gosec if keyjson, err = EncryptKey(key, password, veryLightScryptN, veryLightScryptP); err != nil { t.Errorf("test %d: failed to recrypt key %v", i, err) } diff --git a/accounts/url_test.go b/accounts/url_test.go index bd6f35fa2a0ea..239aa06d227ba 100644 --- a/accounts/url_test.go +++ b/accounts/url_test.go @@ -32,9 +32,10 @@ func TestURLParsing(t *testing.T) { t.Errorf("expected: %v, got: %v", "ethereum.org", url.Path) } - _, err = parseURL("ethereum.org") - if err == nil { - t.Error("expected err, got: nil") + for _, u := range []string{"ethereum.org", ""} { + if _, err = parseURL(u); err == nil { + t.Errorf("input %v, expected err, got: nil", u) + } } } diff --git a/cmd/abigen/main.go b/cmd/abigen/main.go index 8f255143c52fa..5a93f2a92294d 100644 --- a/cmd/abigen/main.go +++ b/cmd/abigen/main.go @@ -55,7 +55,7 @@ var ( } jsonFlag = cli.StringFlag{ Name: "combined-json", - Usage: "Path to the combined-json file generated by compiler", + Usage: "Path to the combined-json file generated by compiler, - for STDIN", } excFlag = cli.StringFlag{ Name: "exc", @@ -165,9 +165,18 @@ func abigen(c *cli.Context) error { var contracts map[string]*compiler.Contract if c.GlobalIsSet(jsonFlag.Name) { - jsonOutput, err := os.ReadFile(c.GlobalString(jsonFlag.Name)) + var ( + input = c.GlobalString(jsonFlag.Name) + jsonOutput []byte + err error + ) + if input == "-" { + jsonOutput, err = io.ReadAll(os.Stdin) + } else { + jsonOutput, err = os.ReadFile(input) + } if err != nil { - utils.Fatalf("Failed to read combined-json from compiler: %v", err) + utils.Fatalf("Failed to read combined-json: %v", err) } contracts, err = compiler.ParseCombinedJSON(jsonOutput, "", "", "", "") if err != nil { @@ -189,7 +198,11 @@ func abigen(c *cli.Context) error { nameParts := strings.Split(name, ":") types = append(types, nameParts[len(nameParts)-1]) - libPattern := crypto.Keccak256Hash([]byte(name)).String()[2:36] + // Derive the library placeholder which is a 34 character prefix of the + // hex encoding of the keccak256 hash of the fully qualified library name. + // Note that the fully qualified library name is the path of its source + // file and the library name separated by ":". + libPattern := crypto.Keccak256Hash([]byte(name)).String()[2:36] // the first 2 chars are 0x libs[libPattern] = nameParts[len(nameParts)-1] } } diff --git a/cmd/devp2p/internal/ethtest/chain.go b/cmd/devp2p/internal/ethtest/chain.go index c1d696b40728f..0a17252a35036 100644 --- a/cmd/devp2p/internal/ethtest/chain.go +++ b/cmd/devp2p/internal/ethtest/chain.go @@ -47,7 +47,7 @@ func (c *Chain) Len() int { // TD calculates the total difficulty of the chain at the // chain head. func (c *Chain) TD() *big.Int { - sum := big.NewInt(0) + sum := new(big.Int) for _, block := range c.blocks[:c.Len()] { sum.Add(sum, block.Difficulty()) } @@ -57,7 +57,7 @@ func (c *Chain) TD() *big.Int { // TotalDifficultyAt calculates the total difficulty of the chain // at the given block height. func (c *Chain) TotalDifficultyAt(height int) *big.Int { - sum := big.NewInt(0) + sum := new(big.Int) if height >= c.Len() { return sum } diff --git a/cmd/devp2p/internal/ethtest/snap.go b/cmd/devp2p/internal/ethtest/snap.go index 2bfd29c75abf1..9c3e88f9cb667 100644 --- a/cmd/devp2p/internal/ethtest/snap.go +++ b/cmd/devp2p/internal/ethtest/snap.go @@ -104,6 +104,7 @@ func (s *Suite) TestSnapGetAccountRange(t *utesting.T) { // Max bytes: 0. Expect to deliver one account. {0, root, zero, ffHash, 1, firstKey, firstKey}, } { + tc := tc if err := s.snapGetAccountRange(t, &tc); err != nil { t.Errorf("test %d \n root: %x\n range: %#x - %#x\n bytes: %d\nfailed: %v", i, tc.root, tc.origin, tc.limit, tc.nBytes, err) } @@ -194,6 +195,7 @@ func (s *Suite) TestSnapGetStorageRanges(t *utesting.T) { expSlots: 2, }, } { + tc := tc if err := s.snapGetStorageRanges(t, &tc); err != nil { t.Errorf("test %d \n root: %x\n range: %#x - %#x\n bytes: %d\n #accounts: %d\nfailed: %v", i, tc.root, tc.origin, tc.limit, tc.nBytes, len(tc.accounts), err) @@ -291,6 +293,7 @@ func (s *Suite) TestSnapGetByteCodes(t *utesting.T) { expHashes: 4, }, } { + tc := tc if err := s.snapGetByteCodes(t, &tc); err != nil { t.Errorf("test %d \n bytes: %d\n #hashes: %d\nfailed: %v", i, tc.nBytes, len(tc.hashes), err) } @@ -436,6 +439,7 @@ func (s *Suite) TestSnapTrieNodes(t *utesting.T) { }, }, } { + tc := tc if err := s.snapGetTrieNodes(t, &tc); err != nil { t.Errorf("test %d \n #hashes %x\n root: %#x\n bytes: %d\nfailed: %v", i, len(tc.expHashes), tc.root, tc.nBytes, err) } diff --git a/cmd/devp2p/internal/v5test/framework.go b/cmd/devp2p/internal/v5test/framework.go index 9eac37520f7b7..6ccbbd075bf0f 100644 --- a/cmd/devp2p/internal/v5test/framework.go +++ b/cmd/devp2p/internal/v5test/framework.go @@ -60,11 +60,9 @@ type conn struct { remoteAddr *net.UDPAddr listeners []net.PacketConn - log logger - codec *v5wire.Codec - lastRequest v5wire.Packet - lastChallenge *v5wire.Whoareyou - idCounter uint32 + log logger + codec *v5wire.Codec + idCounter uint32 } type logger interface { diff --git a/cmd/ethkey/message.go b/cmd/ethkey/message.go index 1a58eeb536fa5..8d253783ceb1b 100644 --- a/cmd/ethkey/message.go +++ b/cmd/ethkey/message.go @@ -21,6 +21,7 @@ import ( "fmt" "os" + "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/accounts/keystore" "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/common" @@ -68,7 +69,7 @@ To sign a message contained in a file, use the --msgfile flag. utils.Fatalf("Error decrypting key: %v", err) } - signature, err := crypto.Sign(signHash(message), key.PrivateKey) + signature, err := crypto.Sign(accounts.TextHash(message), key.PrivateKey) if err != nil { utils.Fatalf("Failed to sign message: %v", err) } @@ -113,7 +114,7 @@ It is possible to refer to a file containing the message.`, utils.Fatalf("Signature encoding is not hexadecimal: %v", err) } - recoveredPubkey, err := crypto.SigToPub(signHash(message), signature) + recoveredPubkey, err := crypto.SigToPub(accounts.TextHash(message), signature) if err != nil || recoveredPubkey == nil { utils.Fatalf("Signature verification failed: %v", err) } diff --git a/cmd/ethkey/utils.go b/cmd/ethkey/utils.go index b81e70913b5b4..ed492272e33f1 100644 --- a/cmd/ethkey/utils.go +++ b/cmd/ethkey/utils.go @@ -23,7 +23,6 @@ import ( "strings" "github.com/ethereum/go-ethereum/cmd/utils" - "github.com/ethereum/go-ethereum/crypto" "gopkg.in/urfave/cli.v1" ) @@ -46,18 +45,6 @@ func getPassphrase(ctx *cli.Context, confirmation bool) string { return utils.GetPassPhrase("", confirmation) } -// signHash is a helper function that calculates a hash for the given message -// that can be safely used to calculate a signature from. -// -// The hash is calculated as -// keccak256("\x19Ethereum Signed Message:\n"${message length}${message}). -// -// This gives context to the signed message and prevents signing of transactions. -func signHash(data []byte) []byte { - msg := fmt.Sprintf("\x19Ethereum Signed Message:\n%d%s", len(data), data) - return crypto.Keccak256([]byte(msg)) -} - // mustPrintJSON prints the JSON encoding of the given object and // exits the program with an error message when the marshaling fails. func mustPrintJSON(jsonObject interface{}) { diff --git a/cmd/evm/internal/t8ntool/execution.go b/cmd/evm/internal/t8ntool/execution.go index 83a0025344a48..241b57f55ea92 100644 --- a/cmd/evm/internal/t8ntool/execution.go +++ b/cmd/evm/internal/t8ntool/execution.go @@ -241,7 +241,7 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, minerReward.Add(minerReward, perOmmer) // Add (8-delta)/8 reward := big.NewInt(8) - reward.Sub(reward, big.NewInt(0).SetUint64(ommer.Delta)) + reward.Sub(reward, new(big.Int).SetUint64(ommer.Delta)) reward.Mul(reward, blockReward) reward.Div(reward, big.NewInt(8)) statedb.AddBalance(ommer.Address, reward) diff --git a/cmd/evm/internal/t8ntool/flags.go b/cmd/evm/internal/t8ntool/flags.go index de666f1151231..12a7002c69ed0 100644 --- a/cmd/evm/internal/t8ntool/flags.go +++ b/cmd/evm/internal/t8ntool/flags.go @@ -152,7 +152,7 @@ var ( "\n\tSyntax (+ExtraEip)", strings.Join(tests.AvailableForks(), "\n\t "), strings.Join(vm.ActivateableEips(), ", ")), - Value: "ArrowGlacier", + Value: "GrayGlacier", } VerbosityFlag = cli.IntFlag{ Name: "verbosity", diff --git a/cmd/evm/t8n_test.go b/cmd/evm/t8n_test.go index 92c01398ba364..805eea90c45e3 100644 --- a/cmd/evm/t8n_test.go +++ b/cmd/evm/t8n_test.go @@ -211,6 +211,14 @@ func TestT8n(t *testing.T) { output: t8nOutput{result: true}, expOut: "exp_arrowglacier.json", }, + { // Difficulty calculation on gray glacier + base: "./testdata/19", + input: t8nInput{ + "alloc.json", "txs.json", "env.json", "GrayGlacier", "", + }, + output: t8nOutput{result: true}, + expOut: "exp_grayglacier.json", + }, { // Sign unprotected (pre-EIP155) transaction base: "./testdata/23", input: t8nInput{ diff --git a/cmd/evm/testdata/19/exp_grayglacier.json b/cmd/evm/testdata/19/exp_grayglacier.json new file mode 100644 index 0000000000000..95a3cb1685cfe --- /dev/null +++ b/cmd/evm/testdata/19/exp_grayglacier.json @@ -0,0 +1,12 @@ +{ + "result": { + "stateRoot": "0x6f058887ca01549716789c380ede95aecc510e6d1fdc4dbf67d053c7c07f4bdc", + "txRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "receiptsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "logsHash": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "receipts": [], + "currentDifficulty": "0x2000000004000", + "gasUsed": "0x0" + } +} \ No newline at end of file diff --git a/cmd/evm/testdata/19/readme.md b/cmd/evm/testdata/19/readme.md index 5fae183f48860..095d4525d4fe9 100644 --- a/cmd/evm/testdata/19/readme.md +++ b/cmd/evm/testdata/19/readme.md @@ -1,9 +1,9 @@ ## Difficulty calculation This test shows how the `evm t8n` can be used to calculate the (ethash) difficulty, if none is provided by the caller, -this time on `ArrowGlacier` (Eip 4345). +this time on `GrayGlacier` (Eip 5133). -Calculating it (with an empty set of txs) using `ArrowGlacier` rules (and no provided unclehash for the parent block): +Calculating it (with an empty set of txs) using `GrayGlacier` rules (and no provided unclehash for the parent block): ``` -[user@work evm]$ ./evm t8n --input.alloc=./testdata/14/alloc.json --input.txs=./testdata/14/txs.json --input.env=./testdata/14/env.json --output.result=stdout --state.fork=ArrowGlacier +[user@work evm]$ ./evm t8n --input.alloc=./testdata/19/alloc.json --input.txs=./testdata/19/txs.json --input.env=./testdata/19/env.json --output.result=stdout --state.fork=GrayGlacier ``` \ No newline at end of file diff --git a/cmd/geth/accountcmd.go b/cmd/geth/accountcmd.go index 0b7d58e8888cd..f065b6eeab148 100644 --- a/cmd/geth/accountcmd.go +++ b/cmd/geth/accountcmd.go @@ -239,14 +239,15 @@ func ambiguousAddrRecovery(ks *keystore.KeyStore, err *keystore.AmbiguousAddrErr } fmt.Println("Testing your password against all of them...") var match *accounts.Account - for _, a := range err.Matches { - if err := ks.Unlock(a, auth); err == nil { - match = &a + for i, a := range err.Matches { + if e := ks.Unlock(a, auth); e == nil { + match = &err.Matches[i] break } } if match == nil { utils.Fatalf("None of the listed files could be unlocked.") + return accounts.Account{} } fmt.Printf("Your password unlocked %s\n", match.URL) fmt.Println("In order to avoid this warning, you need to remove the following duplicate key files:") diff --git a/cmd/geth/config.go b/cmd/geth/config.go index 0946dc89a6f66..b8e80c397f098 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -167,8 +167,8 @@ func makeConfigNode(ctx *cli.Context) (*node.Node, gethConfig) { // makeFullNode loads geth configuration and creates the Ethereum backend. func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) { stack, cfg := makeConfigNode(ctx) - if ctx.GlobalIsSet(utils.OverrideArrowGlacierFlag.Name) { - cfg.Eth.OverrideArrowGlacier = new(big.Int).SetUint64(ctx.GlobalUint64(utils.OverrideArrowGlacierFlag.Name)) + if ctx.GlobalIsSet(utils.OverrideGrayGlacierFlag.Name) { + cfg.Eth.OverrideGrayGlacier = new(big.Int).SetUint64(ctx.GlobalUint64(utils.OverrideGrayGlacierFlag.Name)) } if ctx.GlobalIsSet(utils.OverrideTerminalTotalDifficulty.Name) { cfg.Eth.OverrideTerminalTotalDifficulty = utils.GlobalBig(ctx, utils.OverrideTerminalTotalDifficulty.Name) @@ -176,7 +176,7 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) { backend, eth := utils.RegisterEthService(stack, &cfg.Eth) // Warn users to migrate if they have a legacy freezer format. - if eth != nil { + if eth != nil && !ctx.GlobalIsSet(utils.IgnoreLegacyReceiptsFlag.Name) { firstIdx := uint64(0) // Hack to speed up check for mainnet because we know // the first non-empty block. @@ -188,7 +188,8 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) { if err != nil { log.Error("Failed to check db for legacy receipts", "err", err) } else if isLegacy { - log.Warn("Database has receipts with a legacy format. Please run `geth db freezer-migrate`.") + stack.Close() + utils.Fatalf("Database has receipts with a legacy format. Please run `geth db freezer-migrate`.") } } diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index ace2849c9f065..d996d46f9ac15 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -307,7 +307,7 @@ func checkStateContent(ctx *cli.Context) error { start []byte ) if ctx.NArg() > 1 { - return fmt.Errorf("Max 1 argument: %v", ctx.Command.ArgsUsage) + return fmt.Errorf("max 1 argument: %v", ctx.Command.ArgsUsage) } if ctx.NArg() > 0 { if d, err := hexutil.Decode(ctx.Args().First()); err != nil { @@ -332,8 +332,8 @@ func checkStateContent(ctx *cli.Context) error { ) for it.Next() { count++ - v := it.Value() k := it.Key() + v := it.Value() hasher.Reset() hasher.Write(v) hasher.Read(got) @@ -519,7 +519,7 @@ func dbDumpTrie(ctx *cli.Context) error { return err } } - theTrie, err := trie.New(stRoot, trie.NewDatabase(db)) + theTrie, err := trie.New(common.Hash{}, stRoot, trie.NewDatabase(db)) if err != nil { return err } diff --git a/cmd/geth/les_test.go b/cmd/geth/les_test.go index 73cc23e6674fe..56ed874cd64a5 100644 --- a/cmd/geth/les_test.go +++ b/cmd/geth/les_test.go @@ -81,41 +81,6 @@ func (g *gethrpc) getNodeInfo() *p2p.NodeInfo { return g.nodeInfo } -func (g *gethrpc) waitSynced() { - // Check if it's synced now - var result interface{} - g.callRPC(&result, "eth_syncing") - syncing, ok := result.(bool) - if ok && !syncing { - g.geth.Logf("%v already synced", g.name) - return - } - - // Actually wait, subscribe to the event - ch := make(chan interface{}) - sub, err := g.rpc.Subscribe(context.Background(), "eth", ch, "syncing") - if err != nil { - g.geth.Fatalf("%v syncing: %v", g.name, err) - } - defer sub.Unsubscribe() - timeout := time.After(4 * time.Second) - select { - case ev := <-ch: - g.geth.Log("'syncing' event", ev) - syncing, ok := ev.(bool) - if ok && !syncing { - break - } - g.geth.Log("Other 'syncing' event", ev) - case err := <-sub.Err(): - g.geth.Fatalf("%v notification: %v", g.name, err) - break - case <-timeout: - g.geth.Fatalf("%v timeout syncing", g.name) - break - } -} - // ipcEndpoint resolves an IPC endpoint based on a configured value, taking into // account the set data folders as well as the designated platform we're currently // running on. diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 6b7014f936f13..fbc3e314fb5d6 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -69,7 +69,7 @@ var ( utils.NoUSBFlag, utils.USBFlag, utils.SmartCardDaemonPathFlag, - utils.OverrideArrowGlacierFlag, + utils.OverrideGrayGlacierFlag, utils.OverrideTerminalTotalDifficulty, utils.EthashCacheDirFlag, utils.EthashCachesInMemoryFlag, @@ -174,6 +174,7 @@ var ( utils.StateDiffKnownGapsFilePath, utils.StateDiffWaitForSync, utils.StateDiffWatchedAddressesFilePath, + utils.IgnoreLegacyReceiptsFlag, configFileFlag, }, utils.NetworkFlags, utils.DatabasePathFlags) diff --git a/cmd/geth/snapshot.go b/cmd/geth/snapshot.go index 286eeed8ed2d5..e8c2d0f24188c 100644 --- a/cmd/geth/snapshot.go +++ b/cmd/geth/snapshot.go @@ -101,11 +101,23 @@ In other words, this command does the snapshot to trie conversion. Description: ` geth snapshot check-dangling-storage traverses the snap storage data, and verifies that all snapshot storage data has a corresponding account. +`, + }, + { + Name: "inspect-account", + Usage: "Check all snapshot layers for the a specific account", + ArgsUsage: "
", + Action: utils.MigrateFlags(checkAccount), + Category: "MISCELLANEOUS COMMANDS", + Flags: utils.GroupFlags(utils.NetworkFlags, utils.DatabasePathFlags), + Description: ` +geth snapshot inspect-account
checks all snapshot layers and prints out +information about the specified address. `, }, { Name: "traverse-state", - Usage: "Traverse the state with given root hash for verification", + Usage: "Traverse the state with given root hash and perform quick verification", ArgsUsage: "", Action: utils.MigrateFlags(traverseState), Category: "MISCELLANEOUS COMMANDS", @@ -121,7 +133,7 @@ It's also usable without snapshot enabled. }, { Name: "traverse-rawstate", - Usage: "Traverse the state with given root hash for verification", + Usage: "Traverse the state with given root hash and perform detailed verification", ArgsUsage: "", Action: utils.MigrateFlags(traverseRawState), Category: "MISCELLANEOUS COMMANDS", @@ -266,7 +278,7 @@ func traverseState(ctx *cli.Context) error { log.Info("Start traversing the state", "root", root, "number", headBlock.NumberU64()) } triedb := trie.NewDatabase(chaindb) - t, err := trie.NewSecure(root, triedb) + t, err := trie.NewSecure(common.Hash{}, root, triedb) if err != nil { log.Error("Failed to open trie", "root", root, "err", err) return err @@ -287,7 +299,7 @@ func traverseState(ctx *cli.Context) error { return err } if acc.Root != emptyRoot { - storageTrie, err := trie.NewSecure(acc.Root, triedb) + storageTrie, err := trie.NewSecure(common.BytesToHash(accIter.Key), acc.Root, triedb) if err != nil { log.Error("Failed to open storage trie", "root", acc.Root, "err", err) return err @@ -355,7 +367,7 @@ func traverseRawState(ctx *cli.Context) error { log.Info("Start traversing the state", "root", root, "number", headBlock.NumberU64()) } triedb := trie.NewDatabase(chaindb) - t, err := trie.NewSecure(root, triedb) + t, err := trie.NewSecure(common.Hash{}, root, triedb) if err != nil { log.Error("Failed to open trie", "root", root, "err", err) return err @@ -367,6 +379,8 @@ func traverseRawState(ctx *cli.Context) error { codes int lastReport time.Time start = time.Now() + hasher = crypto.NewKeccakState() + got = make([]byte, 32) ) accIter := t.NodeIterator(nil) for accIter.Next(true) { @@ -376,10 +390,18 @@ func traverseRawState(ctx *cli.Context) error { // Check the present for non-empty hash node(embedded node doesn't // have their own hash). if node != (common.Hash{}) { - if !rawdb.HasTrieNode(chaindb, node) { + blob := rawdb.ReadTrieNode(chaindb, node) + if len(blob) == 0 { log.Error("Missing trie node(account)", "hash", node) return errors.New("missing account") } + hasher.Reset() + hasher.Write(blob) + hasher.Read(got) + if !bytes.Equal(got, node.Bytes()) { + log.Error("Invalid trie node(account)", "hash", node.Hex(), "value", blob) + return errors.New("invalid account node") + } } // If it's a leaf node, yes we are touching an account, // dig into the storage trie further. @@ -391,7 +413,7 @@ func traverseRawState(ctx *cli.Context) error { return errors.New("invalid account") } if acc.Root != emptyRoot { - storageTrie, err := trie.NewSecure(acc.Root, triedb) + storageTrie, err := trie.NewSecure(common.BytesToHash(accIter.LeafKey()), acc.Root, triedb) if err != nil { log.Error("Failed to open storage trie", "root", acc.Root, "err", err) return errors.New("missing storage trie") @@ -404,10 +426,18 @@ func traverseRawState(ctx *cli.Context) error { // Check the present for non-empty hash node(embedded node doesn't // have their own hash). if node != (common.Hash{}) { - if !rawdb.HasTrieNode(chaindb, node) { + blob := rawdb.ReadTrieNode(chaindb, node) + if len(blob) == 0 { log.Error("Missing trie node(storage)", "hash", node) return errors.New("missing storage") } + hasher.Reset() + hasher.Write(blob) + hasher.Read(got) + if !bytes.Equal(got, node.Bytes()) { + log.Error("Invalid trie node(storage)", "hash", node.Hex(), "value", blob) + return errors.New("invalid storage node") + } } // Bump the counter if it's leaf node. if storageIter.Leaf() { @@ -517,3 +547,35 @@ func dumpState(ctx *cli.Context) error { "elapsed", common.PrettyDuration(time.Since(start))) return nil } + +// checkAccount iterates the snap data layers, and looks up the given account +// across all layers. +func checkAccount(ctx *cli.Context) error { + if ctx.NArg() != 1 { + return errors.New("need arg") + } + var ( + hash common.Hash + addr common.Address + ) + switch len(ctx.Args()[0]) { + case 40, 42: + addr = common.HexToAddress(ctx.Args()[0]) + hash = crypto.Keccak256Hash(addr.Bytes()) + case 64, 66: + hash = common.HexToHash(ctx.Args()[0]) + default: + return errors.New("malformed address or hash") + } + stack, _ := makeConfigNode(ctx) + defer stack.Close() + chaindb := utils.MakeChainDatabase(ctx, stack, true) + defer chaindb.Close() + start := time.Now() + log.Info("Checking difflayer journal", "address", addr, "hash", hash) + if err := snapshot.CheckJournalAccount(chaindb, hash); err != nil { + return err + } + log.Info("Checked the snapshot journalled storage", "time", common.PrettyDuration(time.Since(start))) + return nil +} diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 67b8b9d0ae68b..fd0e022745027 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -255,6 +255,7 @@ var AppHelpFlagGroups = []flags.FlagGroup{ Flags: []cli.Flag{ utils.SnapshotFlag, utils.BloomFilterSizeFlag, + utils.IgnoreLegacyReceiptsFlag, cli.HelpFlag, }, }, diff --git a/cmd/puppeth/ssh.go b/cmd/puppeth/ssh.go index 95a36f327236b..0c23ab5562283 100644 --- a/cmd/puppeth/ssh.go +++ b/cmd/puppeth/ssh.go @@ -30,7 +30,7 @@ import ( "github.com/ethereum/go-ethereum/log" "golang.org/x/crypto/ssh" "golang.org/x/crypto/ssh/agent" - "golang.org/x/crypto/ssh/terminal" + "golang.org/x/term" ) // sshClient is a small wrapper around Go's SSH client with a few utility methods @@ -101,7 +101,7 @@ func dial(server string, pubkey []byte) (*sshClient, error) { key, err := ssh.ParsePrivateKey(buf) if err != nil { fmt.Printf("What's the decryption password for %s? (won't be echoed)\n>", path) - blob, err := terminal.ReadPassword(int(os.Stdin.Fd())) + blob, err := term.ReadPassword(int(os.Stdin.Fd())) fmt.Println() if err != nil { log.Warn("Couldn't read password", "err", err) @@ -118,7 +118,7 @@ func dial(server string, pubkey []byte) (*sshClient, error) { } auths = append(auths, ssh.PasswordCallback(func() (string, error) { fmt.Printf("What's the login password for %s at %s? (won't be echoed)\n> ", username, server) - blob, err := terminal.ReadPassword(int(os.Stdin.Fd())) + blob, err := term.ReadPassword(int(os.Stdin.Fd())) fmt.Println() return string(blob), err diff --git a/cmd/puppeth/wizard.go b/cmd/puppeth/wizard.go index f7aafd4dd90a9..6e5ca41d68fab 100644 --- a/cmd/puppeth/wizard.go +++ b/cmd/puppeth/wizard.go @@ -34,7 +34,7 @@ import ( "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/log" "github.com/peterh/liner" - "golang.org/x/crypto/ssh/terminal" + "golang.org/x/term" ) // config contains all the configurations needed by puppeth that should be saved @@ -228,7 +228,7 @@ func (w *wizard) readDefaultFloat(def float64) float64 { // line and returns it. The input will not be echoed. func (w *wizard) readPassword() string { fmt.Printf("> ") - text, err := terminal.ReadPassword(int(os.Stdin.Fd())) + text, err := term.ReadPassword(int(os.Stdin.Fd())) if err != nil { log.Crit("Failed to read password", "err", err) } diff --git a/cmd/utils/diskusage.go b/cmd/utils/diskusage.go index 14cd5cd0bef86..0e88f91944302 100644 --- a/cmd/utils/diskusage.go +++ b/cmd/utils/diskusage.go @@ -33,6 +33,7 @@ func getFreeDiskSpace(path string) (uint64, error) { // Available blocks * size per block = available space in bytes var bavail = stat.Bavail + // nolint:staticcheck if stat.Bavail < 0 { // FreeBSD can have a negative number of blocks available // because of the grace limit. diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index d33702e9b90dd..25581f8ce6c61 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -152,7 +152,7 @@ var ( } RopstenFlag = cli.BoolFlag{ Name: "ropsten", - Usage: "Ropsten network: pre-configured proof-of-work test network", + Usage: "Ropsten network: pre-configured proof-of-stake test network", } RinkebyFlag = cli.BoolFlag{ Name: "rinkeby", @@ -259,9 +259,9 @@ var ( Usage: "Megabytes of memory allocated to bloom-filter for pruning", Value: 2048, } - OverrideArrowGlacierFlag = cli.Uint64Flag{ - Name: "override.arrowglacier", - Usage: "Manually specify Arrow Glacier fork-block, overriding the bundled setting", + OverrideGrayGlacierFlag = cli.Uint64Flag{ + Name: "override.grayglacier", + Usage: "Manually specify Gray Glacier fork-block, overriding the bundled setting", } OverrideTerminalTotalDifficulty = BigFlag{ Name: "override.terminaltotaldifficulty", @@ -568,6 +568,10 @@ var ( Name: "nocompaction", Usage: "Disables db compaction after import", } + IgnoreLegacyReceiptsFlag = cli.BoolFlag{ + Name: "ignore-legacy-receipts", + Usage: "Geth will start up even if there are legacy receipts in freezer", + } // RPC settings IPCDisabledFlag = cli.BoolFlag{ Name: "ipcdisable", diff --git a/common/format.go b/common/format.go index 6fc21af719236..7af41f52d5401 100644 --- a/common/format.go +++ b/common/format.go @@ -27,12 +27,12 @@ import ( // the unnecessary precision off from the formatted textual representation. type PrettyDuration time.Duration -var prettyDurationRe = regexp.MustCompile(`\.[0-9]+`) +var prettyDurationRe = regexp.MustCompile(`\.[0-9]{4,}`) // String implements the Stringer interface, allowing pretty printing of duration // values rounded to three decimals. func (d PrettyDuration) String() string { - label := fmt.Sprintf("%v", time.Duration(d)) + label := time.Duration(d).String() if match := prettyDurationRe.FindString(label); len(match) > 4 { label = strings.Replace(label, match, match[:4], 1) } diff --git a/consensus/clique/clique.go b/consensus/clique/clique.go index 685186817d2d0..950300f034868 100644 --- a/consensus/clique/clique.go +++ b/consensus/clique/clique.go @@ -180,7 +180,7 @@ type Clique struct { signer common.Address // Ethereum address of the signing key signFn SignerFn // Signer function to authorize hashes with - lock sync.RWMutex // Protects the signer fields + lock sync.RWMutex // Protects the signer and proposals fields // The fields below are for testing only fakeDiff bool // Skip difficulty verifications @@ -507,9 +507,8 @@ func (c *Clique) Prepare(chain consensus.ChainHeaderReader, header *types.Header if err != nil { return err } + c.lock.RLock() if number%c.config.Epoch != 0 { - c.lock.RLock() - // Gather all the proposals that make sense voting on addresses := make([]common.Address, 0, len(c.proposals)) for address, authorize := range c.proposals { @@ -526,10 +525,14 @@ func (c *Clique) Prepare(chain consensus.ChainHeaderReader, header *types.Header copy(header.Nonce[:], nonceDropVote) } } - c.lock.RUnlock() } + + // Copy signer protected by mutex to avoid race condition + signer := c.signer + c.lock.RUnlock() + // Set the correct difficulty - header.Difficulty = calcDifficulty(snap, c.signer) + header.Difficulty = calcDifficulty(snap, signer) // Ensure the extra data has all its components if len(header.Extra) < extraVanity { @@ -666,7 +669,10 @@ func (c *Clique) CalcDifficulty(chain consensus.ChainHeaderReader, time uint64, if err != nil { return nil } - return calcDifficulty(snap, c.signer) + c.lock.RLock() + signer := c.signer + c.lock.RUnlock() + return calcDifficulty(snap, signer) } func calcDifficulty(snap *Snapshot, signer common.Address) *big.Int { diff --git a/consensus/ethash/consensus.go b/consensus/ethash/consensus.go index 6a93fead29db6..1c38b80ea59b7 100644 --- a/consensus/ethash/consensus.go +++ b/consensus/ethash/consensus.go @@ -45,6 +45,11 @@ var ( maxUncles = 2 // Maximum number of uncles allowed in a single block allowedFutureBlockTimeSeconds = int64(15) // Max seconds from current time allowed for blocks, before they're considered future blocks + // calcDifficultyEip5133 is the difficulty adjustment algorithm as specified by EIP 5133. + // It offsets the bomb a total of 11.4M blocks. + // Specification EIP-5133: https://eips.ethereum.org/EIPS/eip-5133 + calcDifficultyEip5133 = makeDifficultyCalculator(big.NewInt(11_400_000)) + // calcDifficultyEip4345 is the difficulty adjustment algorithm as specified by EIP 4345. // It offsets the bomb a total of 10.7M blocks. // Specification EIP-4345: https://eips.ethereum.org/EIPS/eip-4345 @@ -334,6 +339,8 @@ func (ethash *Ethash) CalcDifficulty(chain consensus.ChainHeaderReader, time uin func CalcDifficulty(config *params.ChainConfig, time uint64, parent *types.Header) *big.Int { next := new(big.Int).Add(parent.Number, big1) switch { + case config.IsGrayGlacier(next): + return calcDifficultyEip5133(time, parent) case config.IsArrowGlacier(next): return calcDifficultyEip4345(time, parent) case config.IsLondon(next): diff --git a/consensus/ethash/consensus_test.go b/consensus/ethash/consensus_test.go index bca424af30e36..db997d737e625 100644 --- a/consensus/ethash/consensus_test.go +++ b/consensus/ethash/consensus_test.go @@ -103,7 +103,7 @@ func TestDifficultyCalculators(t *testing.T) { for i := 0; i < 5000; i++ { // 1 to 300 seconds diff var timeDelta = uint64(1 + rand.Uint32()%3000) - diffBig := big.NewInt(0).SetBytes(randSlice(2, 10)) + diffBig := new(big.Int).SetBytes(randSlice(2, 10)) if diffBig.Cmp(params.MinimumDifficulty) < 0 { diffBig.Set(params.MinimumDifficulty) } diff --git a/consensus/misc/eip1559.go b/consensus/misc/eip1559.go index e18340b0f33ec..e0216243f0572 100644 --- a/consensus/misc/eip1559.go +++ b/consensus/misc/eip1559.go @@ -58,36 +58,36 @@ func CalcBaseFee(config *params.ChainConfig, parent *types.Header) *big.Int { return new(big.Int).SetUint64(params.InitialBaseFee) } - var ( - parentGasTarget = parent.GasLimit / params.ElasticityMultiplier - parentGasTargetBig = new(big.Int).SetUint64(parentGasTarget) - baseFeeChangeDenominator = new(big.Int).SetUint64(params.BaseFeeChangeDenominator) - ) + parentGasTarget := parent.GasLimit / params.ElasticityMultiplier // If the parent gasUsed is the same as the target, the baseFee remains unchanged. if parent.GasUsed == parentGasTarget { return new(big.Int).Set(parent.BaseFee) } + + var ( + num = new(big.Int) + denom = new(big.Int) + ) + if parent.GasUsed > parentGasTarget { // If the parent block used more gas than its target, the baseFee should increase. - gasUsedDelta := new(big.Int).SetUint64(parent.GasUsed - parentGasTarget) - x := new(big.Int).Mul(parent.BaseFee, gasUsedDelta) - y := x.Div(x, parentGasTargetBig) - baseFeeDelta := math.BigMax( - x.Div(y, baseFeeChangeDenominator), - common.Big1, - ) + // max(1, parentBaseFee * gasUsedDelta / parentGasTarget / baseFeeChangeDenominator) + num.SetUint64(parent.GasUsed - parentGasTarget) + num.Mul(num, parent.BaseFee) + num.Div(num, denom.SetUint64(parentGasTarget)) + num.Div(num, denom.SetUint64(params.BaseFeeChangeDenominator)) + baseFeeDelta := math.BigMax(num, common.Big1) - return x.Add(parent.BaseFee, baseFeeDelta) + return num.Add(parent.BaseFee, baseFeeDelta) } else { // Otherwise if the parent block used less gas than its target, the baseFee should decrease. - gasUsedDelta := new(big.Int).SetUint64(parentGasTarget - parent.GasUsed) - x := new(big.Int).Mul(parent.BaseFee, gasUsedDelta) - y := x.Div(x, parentGasTargetBig) - baseFeeDelta := x.Div(y, baseFeeChangeDenominator) + // max(0, parentBaseFee * gasUsedDelta / parentGasTarget / baseFeeChangeDenominator) + num.SetUint64(parentGasTarget - parent.GasUsed) + num.Mul(num, parent.BaseFee) + num.Div(num, denom.SetUint64(parentGasTarget)) + num.Div(num, denom.SetUint64(params.BaseFeeChangeDenominator)) + baseFee := num.Sub(parent.BaseFee, num) - return math.BigMax( - x.Sub(parent.BaseFee, baseFeeDelta), - common.Big0, - ) + return math.BigMax(baseFee, common.Big0) } } diff --git a/contracts/checkpointoracle/oracle.go b/contracts/checkpointoracle/oracle.go index eff8ce2f2b803..dec01db24435c 100644 --- a/contracts/checkpointoracle/oracle.go +++ b/contracts/checkpointoracle/oracle.go @@ -17,7 +17,8 @@ // Package checkpointoracle is a an on-chain light client checkpoint oracle. package checkpointoracle -//go:generate go run ../../cmd/abigen --sol contract/oracle.sol --pkg contract --out contract/oracle.go +//go:generate solc contract/oracle.sol --combined-json bin,bin-runtime,srcmap,srcmap-runtime,abi,userdoc,devdoc,metadata,hashes --optimize -o ./ --overwrite +//go:generate go run ../../cmd/abigen --pkg contract --out contract/oracle.go --combined-json ./combined.json import ( "errors" diff --git a/core/beacon/types.go b/core/beacon/types.go index 18d5d2ab78b46..97bf66cd3fe41 100644 --- a/core/beacon/types.go +++ b/core/beacon/types.go @@ -148,6 +148,13 @@ func ExecutableDataToBlock(params ExecutableDataV1) (*types.Block, error) { if len(params.ExtraData) > 32 { return nil, fmt.Errorf("invalid extradata length: %v", len(params.ExtraData)) } + if len(params.LogsBloom) != 256 { + return nil, fmt.Errorf("invalid logsBloom length: %v", len(params.LogsBloom)) + } + // Check that baseFeePerGas is not negative or too big + if params.BaseFeePerGas != nil && (params.BaseFeePerGas.Sign() == -1 || params.BaseFeePerGas.BitLen() > 256) { + return nil, fmt.Errorf("invalid baseFeePerGas: %v", params.BaseFeePerGas) + } header := &types.Header{ ParentHash: params.ParentHash, UncleHash: types.EmptyUncleHash, diff --git a/core/blockchain.go b/core/blockchain.go index 747fc500d4bc5..3c5d44f468238 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -681,7 +681,7 @@ func (bc *BlockChain) SnapSyncCommitHead(hash common.Hash) error { if block == nil { return fmt.Errorf("non existent block [%x..]", hash[:4]) } - if _, err := trie.NewSecure(block.Root(), bc.stateCache.TrieDB()); err != nil { + if _, err := trie.NewSecure(common.Hash{}, block.Root(), bc.stateCache.TrieDB()); err != nil { return err } @@ -1984,8 +1984,8 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error { oldChain types.Blocks commonBlock *types.Block - deletedTxs types.Transactions - addedTxs types.Transactions + deletedTxs []common.Hash + addedTxs []common.Hash deletedLogs [][]*types.Log rebirthLogs [][]*types.Log @@ -1995,7 +1995,9 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error { // Old chain is longer, gather all transactions and logs as deleted ones for ; oldBlock != nil && oldBlock.NumberU64() != newBlock.NumberU64(); oldBlock = bc.GetBlock(oldBlock.ParentHash(), oldBlock.NumberU64()-1) { oldChain = append(oldChain, oldBlock) - deletedTxs = append(deletedTxs, oldBlock.Transactions()...) + for _, tx := range oldBlock.Transactions() { + deletedTxs = append(deletedTxs, tx.Hash()) + } // Collect deleted logs for notification logs := bc.collectLogs(oldBlock.Hash(), true) @@ -2025,7 +2027,9 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error { } // Remove an old block as well as stash away a new block oldChain = append(oldChain, oldBlock) - deletedTxs = append(deletedTxs, oldBlock.Transactions()...) + for _, tx := range oldBlock.Transactions() { + deletedTxs = append(deletedTxs, tx.Hash()) + } // Collect deleted logs for notification logs := bc.collectLogs(oldBlock.Hash(), true) @@ -2044,6 +2048,7 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error { return fmt.Errorf("invalid new chain") } } + // Ensure the user sees large reorgs if len(oldChain) > 0 && len(newChain) > 0 { logFn := log.Info @@ -2060,7 +2065,7 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error { } else if len(newChain) > 0 { // Special case happens in the post merge stage that current head is // the ancestor of new head while these two blocks are not consecutive - log.Info("Extend chain", "add", len(newChain), "number", newChain[0].NumberU64(), "hash", newChain[0].Hash()) + log.Info("Extend chain", "add", len(newChain), "number", newChain[0].Number(), "hash", newChain[0].Hash()) blockReorgAddMeter.Mark(int64(len(newChain))) } else { // len(newChain) == 0 && len(oldChain) > 0 @@ -2073,22 +2078,26 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error { // Insert the block in the canonical way, re-writing history bc.writeHeadBlock(newChain[i]) - // Collect reborn logs due to chain reorg - logs := bc.collectLogs(newChain[i].Hash(), false) - if len(logs) > 0 { - rebirthLogs = append(rebirthLogs, logs) - } // Collect the new added transactions. - addedTxs = append(addedTxs, newChain[i].Transactions()...) + for _, tx := range newChain[i].Transactions() { + addedTxs = append(addedTxs, tx.Hash()) + } } + // Delete useless indexes right now which includes the non-canonical // transaction indexes, canonical chain indexes which above the head. indexesBatch := bc.db.NewBatch() - for _, tx := range types.TxDifference(deletedTxs, addedTxs) { - rawdb.DeleteTxLookupEntry(indexesBatch, tx.Hash()) + for _, tx := range types.HashDifference(deletedTxs, addedTxs) { + rawdb.DeleteTxLookupEntry(indexesBatch, tx) + } + + // Delete all hash markers that are not part of the new canonical chain. + // Because the reorg function does not handle new chain head, all hash + // markers greater than or equal to new chain head should be deleted. + number := commonBlock.NumberU64() + if len(newChain) > 1 { + number = newChain[1].NumberU64() } - // Delete any canonical number assignments above the new head - number := bc.CurrentBlock().NumberU64() for i := number + 1; ; i++ { hash := rawdb.ReadCanonicalHash(bc.db, i) if hash == (common.Hash{}) { @@ -2099,6 +2108,15 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error { if err := indexesBatch.Write(); err != nil { log.Crit("Failed to delete useless indexes", "err", err) } + + // Collect the logs + for i := len(newChain) - 1; i >= 1; i-- { + // Collect reborn logs due to chain reorg + logs := bc.collectLogs(newChain[i].Hash(), false) + if len(logs) > 0 { + rebirthLogs = append(rebirthLogs, logs) + } + } // If any logs need to be fired, do it now. In theory we could avoid creating // this goroutine if there are no events to fire, but realistcally that only // ever happens if we're reorging empty blocks, which will only happen on idle diff --git a/core/blockchain_sethead_test.go b/core/blockchain_sethead_test.go index 970e0306308dd..d8103b5b5c50e 100644 --- a/core/blockchain_sethead_test.go +++ b/core/blockchain_sethead_test.go @@ -51,6 +51,7 @@ type rewindTest struct { expHeadBlock uint64 // Block number of the expected head full block } +//nolint:unused func (tt *rewindTest) dump(crash bool) string { buffer := new(strings.Builder) diff --git a/core/blockchain_snapshot_test.go b/core/blockchain_snapshot_test.go index dfa8ed65ec6d5..189eee59a009a 100644 --- a/core/blockchain_snapshot_test.go +++ b/core/blockchain_snapshot_test.go @@ -150,6 +150,7 @@ func (basic *snapshotTestBasic) verify(t *testing.T, chain *BlockChain, blocks [ } } +//nolint:unused func (basic *snapshotTestBasic) dump() string { buffer := new(strings.Builder) @@ -341,54 +342,6 @@ func (snaptest *setHeadSnapshotTest) test(t *testing.T) { snaptest.verify(t, newchain, blocks) } -// restartCrashSnapshotTest is the test type used to test this scenario: -// - have a complete snapshot -// - restart chain -// - insert more blocks with enabling the snapshot -// - commit the snapshot -// - crash -// - restart again -type restartCrashSnapshotTest struct { - snapshotTestBasic - newBlocks int -} - -func (snaptest *restartCrashSnapshotTest) test(t *testing.T) { - // It's hard to follow the test case, visualize the input - // log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true)))) - // fmt.Println(tt.dump()) - chain, blocks := snaptest.prepare(t) - - // Firstly, stop the chain properly, with all snapshot journal - // and state committed. - chain.Stop() - - newchain, err := NewBlockChain(snaptest.db, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil) - if err != nil { - t.Fatalf("Failed to recreate chain: %v", err) - } - newBlocks, _ := GenerateChain(params.TestChainConfig, blocks[len(blocks)-1], snaptest.engine, snaptest.gendb, snaptest.newBlocks, func(i int, b *BlockGen) {}) - newchain.InsertChain(newBlocks) - - // Commit the entire snapshot into the disk if requested. Note only - // (a) snapshot root and (b) snapshot generator will be committed, - // the diff journal is not. - newchain.Snapshots().Cap(newBlocks[len(newBlocks)-1].Root(), 0) - - // Simulate the blockchain crash - // Don't call chain.Stop here, so that no snapshot - // journal and latest state will be committed - - // Restart the chain after the crash - newchain, err = NewBlockChain(snaptest.db, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil) - if err != nil { - t.Fatalf("Failed to recreate chain: %v", err) - } - defer newchain.Stop() - - snaptest.verify(t, newchain, blocks) -} - // wipeCrashSnapshotTest is the test type used to test this scenario: // - have a complete snapshot // - restart, insert more blocks without enabling the snapshot @@ -431,7 +384,7 @@ func (snaptest *wipeCrashSnapshotTest) test(t *testing.T) { SnapshotLimit: 256, SnapshotWait: false, // Don't wait rebuild } - newchain, err = NewBlockChain(snaptest.db, config, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil) + _, err = NewBlockChain(snaptest.db, config, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to recreate chain: %v", err) } diff --git a/core/blockchain_test.go b/core/blockchain_test.go index b42f572b12900..c1e1d6371e0da 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -503,7 +503,7 @@ func TestReorgShortBlocks(t *testing.T) { testReorgShort(t, true) } func testReorgShort(t *testing.T, full bool) { // Create a long easy chain vs. a short heavy one. Due to difficulty adjustment // we need a fairly long chain of blocks with different difficulties for a short - // one to become heavyer than a long one. The 96 is an empirical value. + // one to become heavier than a long one. The 96 is an empirical value. easy := make([]int64, 96) for i := 0; i < len(easy); i++ { easy[i] = 60 @@ -2576,6 +2576,7 @@ func TestTransactionIndices(t *testing.T) { t.Fatalf("failed to create temp freezer db: %v", err) } gspec.MustCommit(ancientDb) + l := l chain, err = NewBlockChain(ancientDb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, &l) if err != nil { t.Fatalf("failed to create tester chain: %v", err) @@ -2601,6 +2602,7 @@ func TestTransactionIndices(t *testing.T) { limit = []uint64{0, 64 /* drop stale */, 32 /* shorten history */, 64 /* extend history */, 0 /* restore all */} tails := []uint64{0, 67 /* 130 - 64 + 1 */, 100 /* 131 - 32 + 1 */, 69 /* 132 - 64 + 1 */, 0} for i, l := range limit { + l := l chain, err = NewBlockChain(ancientDb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, &l) if err != nil { t.Fatalf("failed to create tester chain: %v", err) @@ -2762,7 +2764,7 @@ func BenchmarkBlockChain_1x1000ValueTransferToNonexisting(b *testing.B) { numBlocks = 1 ) recipientFn := func(nonce uint64) common.Address { - return common.BigToAddress(big.NewInt(0).SetUint64(1337 + nonce)) + return common.BigToAddress(new(big.Int).SetUint64(1337 + nonce)) } dataFn := func(nonce uint64) []byte { return nil @@ -2779,7 +2781,7 @@ func BenchmarkBlockChain_1x1000ValueTransferToExisting(b *testing.B) { b.ResetTimer() recipientFn := func(nonce uint64) common.Address { - return common.BigToAddress(big.NewInt(0).SetUint64(1337)) + return common.BigToAddress(new(big.Int).SetUint64(1337)) } dataFn := func(nonce uint64) []byte { return nil @@ -2796,7 +2798,7 @@ func BenchmarkBlockChain_1x1000Executions(b *testing.B) { b.ResetTimer() recipientFn := func(nonce uint64) common.Address { - return common.BigToAddress(big.NewInt(0).SetUint64(0xc0de)) + return common.BigToAddress(new(big.Int).SetUint64(0xc0de)) } dataFn := func(nonce uint64) []byte { return nil @@ -3758,3 +3760,112 @@ func TestSetCanonical(t *testing.T) { chain.SetCanonical(canon[TriesInMemory-1]) verify(canon[TriesInMemory-1]) } + +// TestCanonicalHashMarker tests all the canonical hash markers are updated/deleted +// correctly in case reorg is called. +func TestCanonicalHashMarker(t *testing.T) { + var cases = []struct { + forkA int + forkB int + }{ + // ForkA: 10 blocks + // ForkB: 1 blocks + // + // reorged: + // markers [2, 10] should be deleted + // markers [1] should be updated + {10, 1}, + + // ForkA: 10 blocks + // ForkB: 2 blocks + // + // reorged: + // markers [3, 10] should be deleted + // markers [1, 2] should be updated + {10, 2}, + + // ForkA: 10 blocks + // ForkB: 10 blocks + // + // reorged: + // markers [1, 10] should be updated + {10, 10}, + + // ForkA: 10 blocks + // ForkB: 11 blocks + // + // reorged: + // markers [1, 11] should be updated + {10, 11}, + } + for _, c := range cases { + var ( + db = rawdb.NewMemoryDatabase() + gspec = &Genesis{ + Config: params.TestChainConfig, + Alloc: GenesisAlloc{}, + BaseFee: big.NewInt(params.InitialBaseFee), + } + genesis = gspec.MustCommit(db) + engine = ethash.NewFaker() + ) + forkA, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, c.forkA, func(i int, gen *BlockGen) {}) + forkB, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, c.forkB, func(i int, gen *BlockGen) {}) + + // Initialize test chain + diskdb := rawdb.NewMemoryDatabase() + gspec.MustCommit(diskdb) + chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil, nil) + if err != nil { + t.Fatalf("failed to create tester chain: %v", err) + } + // Insert forkA and forkB, the canonical should on forkA still + if n, err := chain.InsertChain(forkA); err != nil { + t.Fatalf("block %d: failed to insert into chain: %v", n, err) + } + if n, err := chain.InsertChain(forkB); err != nil { + t.Fatalf("block %d: failed to insert into chain: %v", n, err) + } + + verify := func(head *types.Block) { + if chain.CurrentBlock().Hash() != head.Hash() { + t.Fatalf("Unexpected block hash, want %x, got %x", head.Hash(), chain.CurrentBlock().Hash()) + } + if chain.CurrentFastBlock().Hash() != head.Hash() { + t.Fatalf("Unexpected fast block hash, want %x, got %x", head.Hash(), chain.CurrentFastBlock().Hash()) + } + if chain.CurrentHeader().Hash() != head.Hash() { + t.Fatalf("Unexpected head header, want %x, got %x", head.Hash(), chain.CurrentHeader().Hash()) + } + if !chain.HasState(head.Root()) { + t.Fatalf("Lost block state %v %x", head.Number(), head.Hash()) + } + } + + // Switch canonical chain to forkB if necessary + if len(forkA) < len(forkB) { + verify(forkB[len(forkB)-1]) + } else { + verify(forkA[len(forkA)-1]) + chain.SetCanonical(forkB[len(forkB)-1]) + verify(forkB[len(forkB)-1]) + } + + // Ensure all hash markers are updated correctly + for i := 0; i < len(forkB); i++ { + block := forkB[i] + hash := chain.GetCanonicalHash(block.NumberU64()) + if hash != block.Hash() { + t.Fatalf("Unexpected canonical hash %d", block.NumberU64()) + } + } + if c.forkA > c.forkB { + for i := uint64(c.forkB) + 1; i <= uint64(c.forkA); i++ { + hash := chain.GetCanonicalHash(i) + if hash != (common.Hash{}) { + t.Fatalf("Unexpected canonical hash %d", i) + } + } + } + } +} diff --git a/core/bloombits/matcher_test.go b/core/bloombits/matcher_test.go index 923579221f516..93d4632b8587b 100644 --- a/core/bloombits/matcher_test.go +++ b/core/bloombits/matcher_test.go @@ -124,13 +124,13 @@ func makeRandomIndexes(lengths []int, max int) [][]bloomIndexes { // testMatcherDiffBatches runs the given matches test in single-delivery and also // in batches delivery mode, verifying that all kinds of deliveries are handled -// correctly withn. +// correctly within. func testMatcherDiffBatches(t *testing.T, filter [][]bloomIndexes, start, blocks uint64, intermittent bool, retrievals uint32) { singleton := testMatcher(t, filter, start, blocks, intermittent, retrievals, 1) batched := testMatcher(t, filter, start, blocks, intermittent, retrievals, 16) if singleton != batched { - t.Errorf("filter = %v blocks = %v intermittent = %v: request count mismatch, %v in signleton vs. %v in batched mode", filter, blocks, intermittent, singleton, batched) + t.Errorf("filter = %v blocks = %v intermittent = %v: request count mismatch, %v in singleton vs. %v in batched mode", filter, blocks, intermittent, singleton, batched) } } diff --git a/core/evm.go b/core/evm.go index 536ac673e6a62..e929da25eaeea 100644 --- a/core/evm.go +++ b/core/evm.go @@ -31,7 +31,7 @@ type ChainContext interface { // Engine retrieves the chain's consensus engine. Engine() consensus.Engine - // GetHeader returns the hash corresponding to their hash. + // GetHeader returns the header corresponding to the hash/number argument pair. GetHeader(common.Hash, uint64) *types.Header } @@ -84,6 +84,11 @@ func GetHashFn(ref *types.Header, chain ChainContext) func(n uint64) common.Hash var cache []common.Hash return func(n uint64) common.Hash { + if ref.Number.Uint64() <= n { + // This situation can happen if we're doing tracing and using + // block overrides. + return common.Hash{} + } // If there's no hash cache yet, make one if len(cache) == 0 { cache = append(cache, ref.ParentHash) diff --git a/core/forkid/forkid_test.go b/core/forkid/forkid_test.go index b0ee59b9eb7b2..ca698c47171d5 100644 --- a/core/forkid/forkid_test.go +++ b/core/forkid/forkid_test.go @@ -31,7 +31,7 @@ import ( // the correct fork ID. func TestCreation(t *testing.T) { mergeConfig := *params.MainnetChainConfig - mergeConfig.MergeForkBlock = big.NewInt(15000000) + mergeConfig.MergeNetsplitBlock = big.NewInt(18000000) type testcase struct { head uint64 want ID @@ -68,8 +68,10 @@ func TestCreation(t *testing.T) { {12964999, ID{Hash: checksumToBytes(0x0eb440f6), Next: 12965000}}, // Last Berlin block {12965000, ID{Hash: checksumToBytes(0xb715077d), Next: 13773000}}, // First London block {13772999, ID{Hash: checksumToBytes(0xb715077d), Next: 13773000}}, // Last London block - {13773000, ID{Hash: checksumToBytes(0x20c327fc), Next: 0}}, // First Arrow Glacier block - {20000000, ID{Hash: checksumToBytes(0x20c327fc), Next: 0}}, // Future Arrow Glacier block + {13773000, ID{Hash: checksumToBytes(0x20c327fc), Next: 15050000}}, // First Arrow Glacier block + {15049999, ID{Hash: checksumToBytes(0x20c327fc), Next: 15050000}}, // Last Arrow Glacier block + {15050000, ID{Hash: checksumToBytes(0xf0afd0e3), Next: 0}}, // First Gray Glacier block + {20000000, ID{Hash: checksumToBytes(0xf0afd0e3), Next: 0}}, // Future Gray Glacier block }, }, // Ropsten test cases @@ -163,9 +165,11 @@ func TestCreation(t *testing.T) { {12964999, ID{Hash: checksumToBytes(0x0eb440f6), Next: 12965000}}, // Last Berlin block {12965000, ID{Hash: checksumToBytes(0xb715077d), Next: 13773000}}, // First London block {13772999, ID{Hash: checksumToBytes(0xb715077d), Next: 13773000}}, // Last London block - {13773000, ID{Hash: checksumToBytes(0x20c327fc), Next: 15000000}}, // First Arrow Glacier block - {15000000, ID{Hash: checksumToBytes(0xe3abe201), Next: 0}}, // First Merge Start block - {20000000, ID{Hash: checksumToBytes(0xe3abe201), Next: 0}}, // Future Merge Start block + {13773000, ID{Hash: checksumToBytes(0x20c327fc), Next: 15050000}}, // First Arrow Glacier block + {15049999, ID{Hash: checksumToBytes(0x20c327fc), Next: 15050000}}, // Last Arrow Glacier block + {15050000, ID{Hash: checksumToBytes(0xf0afd0e3), Next: 18000000}}, // First Gray Glacier block + {18000000, ID{Hash: checksumToBytes(0x4fb8a872), Next: 0}}, // First Merge Start block + {20000000, ID{Hash: checksumToBytes(0x4fb8a872), Next: 0}}, // Future Merge Start block }, }, } @@ -242,11 +246,11 @@ func TestValidation(t *testing.T) { // Local is mainnet Petersburg, remote is Rinkeby Petersburg. {7987396, ID{Hash: checksumToBytes(0xafec6b27), Next: 0}, ErrLocalIncompatibleOrStale}, - // Local is mainnet Arrow Glacier, far in the future. Remote announces Gopherium (non existing fork) + // Local is mainnet Gray Glacier, far in the future. Remote announces Gopherium (non existing fork) // at some future block 88888888, for itself, but past block for local. Local is incompatible. // // This case detects non-upgraded nodes with majority hash power (typical Ropsten mess). - {88888888, ID{Hash: checksumToBytes(0x20c327fc), Next: 88888888}, ErrLocalIncompatibleOrStale}, + {88888888, ID{Hash: checksumToBytes(0xf0afd0e3), Next: 88888888}, ErrLocalIncompatibleOrStale}, // Local is mainnet Byzantium. Remote is also in Byzantium, but announces Gopherium (non existing // fork) at block 7279999, before Petersburg. Local is incompatible. diff --git a/core/genesis.go b/core/genesis.go index 64ee99c5443de..aa7d704ea2c75 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -233,7 +233,7 @@ func SetupGenesisBlock(db ethdb.Database, genesis *Genesis) (*params.ChainConfig return SetupGenesisBlockWithOverride(db, genesis, nil, nil) } -func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, overrideArrowGlacier, overrideTerminalTotalDifficulty *big.Int) (*params.ChainConfig, common.Hash, error) { +func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, overrideGrayGlacier, overrideTerminalTotalDifficulty *big.Int) (*params.ChainConfig, common.Hash, error) { if genesis != nil && genesis.Config == nil { return params.AllEthashProtocolChanges, common.Hash{}, errGenesisNoConfig } @@ -279,8 +279,8 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, override } // Get the existing chain configuration. newcfg := genesis.configOrDefault(stored) - if overrideArrowGlacier != nil { - newcfg.ArrowGlacierBlock = overrideArrowGlacier + if overrideGrayGlacier != nil { + newcfg.GrayGlacierBlock = overrideGrayGlacier } if overrideTerminalTotalDifficulty != nil { newcfg.TerminalTotalDifficulty = overrideTerminalTotalDifficulty @@ -301,8 +301,8 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, override // apply the overrides. if genesis == nil && stored != params.MainnetGenesisHash { newcfg = storedcfg - if overrideArrowGlacier != nil { - newcfg.ArrowGlacierBlock = overrideArrowGlacier + if overrideGrayGlacier != nil { + newcfg.GrayGlacierBlock = overrideGrayGlacier } if overrideTerminalTotalDifficulty != nil { newcfg.TerminalTotalDifficulty = overrideTerminalTotalDifficulty diff --git a/core/rawdb/freezer_test.go b/core/rawdb/freezer_test.go index c28d35ef387d0..630911ec867c7 100644 --- a/core/rawdb/freezer_test.go +++ b/core/rawdb/freezer_test.go @@ -277,7 +277,7 @@ func TestFreezerReadonlyValidate(t *testing.T) { // Re-openening as readonly should fail when validating // table lengths. - f, err = NewFreezer(dir, "", true, 2049, tables) + _, err = NewFreezer(dir, "", true, 2049, tables) if err == nil { t.Fatal("readonly freezer should fail with differing table lengths") } diff --git a/core/rawdb/freezer_utils_test.go b/core/rawdb/freezer_utils_test.go index cc300cb614fa0..829cbfb4f332d 100644 --- a/core/rawdb/freezer_utils_test.go +++ b/core/rawdb/freezer_utils_test.go @@ -43,7 +43,7 @@ func TestCopyFrom(t *testing.T) { {"foo", "bar", 8, true}, } for _, c := range cases { - os.WriteFile(c.src, content, 0644) + os.WriteFile(c.src, content, 0600) if err := copyFrom(c.src, c.dest, c.offset, func(f *os.File) error { if !c.writePrefix { diff --git a/core/state/database.go b/core/state/database.go index bbcd2358e5b8e..ce5d8d731715f 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -133,7 +133,7 @@ type cachingDB struct { // OpenTrie opens the main account trie at a specific root hash. func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) { - tr, err := trie.NewSecure(root, db.db) + tr, err := trie.NewSecure(common.Hash{}, root, db.db) if err != nil { return nil, err } @@ -142,7 +142,7 @@ func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) { // OpenStorageTrie opens the storage trie of an account. func (db *cachingDB) OpenStorageTrie(addrHash, root common.Hash) (Trie, error) { - tr, err := trie.NewSecure(root, db.db) + tr, err := trie.NewSecure(addrHash, root, db.db) if err != nil { return nil, err } diff --git a/core/state/pruner/bloom.go b/core/state/pruner/bloom.go index 29bc4e7314efa..72315db720f19 100644 --- a/core/state/pruner/bloom.go +++ b/core/state/pruner/bloom.go @@ -39,7 +39,7 @@ func (f stateBloomHasher) BlockSize() int { panic("not implem func (f stateBloomHasher) Size() int { return 8 } func (f stateBloomHasher) Sum64() uint64 { return binary.BigEndian.Uint64(f) } -// stateBloom is a bloom filter used during the state convesion(snapshot->state). +// stateBloom is a bloom filter used during the state conversion(snapshot->state). // The keys of all generated entries will be recorded here so that in the pruning // stage the entries belong to the specific version can be avoided for deletion. // @@ -100,7 +100,7 @@ func (bloom *stateBloom) Commit(filename, tempname string) error { } f.Close() - // Move the teporary file into it's final location + // Move the temporary file into it's final location return os.Rename(tempname, filename) } diff --git a/core/state/pruner/pruner.go b/core/state/pruner/pruner.go index a121839bd099f..2f4b068d88f30 100644 --- a/core/state/pruner/pruner.go +++ b/core/state/pruner/pruner.go @@ -410,7 +410,7 @@ func extractGenesis(db ethdb.Database, stateBloom *stateBloom) error { if genesis == nil { return errors.New("missing genesis block") } - t, err := trie.NewSecure(genesis.Root(), trie.NewDatabase(db)) + t, err := trie.NewSecure(common.Hash{}, genesis.Root(), trie.NewDatabase(db)) if err != nil { return err } @@ -430,7 +430,7 @@ func extractGenesis(db ethdb.Database, stateBloom *stateBloom) error { return err } if acc.Root != emptyRoot { - storageTrie, err := trie.NewSecure(acc.Root, trie.NewDatabase(db)) + storageTrie, err := trie.NewSecure(common.BytesToHash(accIter.LeafKey()), acc.Root, trie.NewDatabase(db)) if err != nil { return err } diff --git a/core/state/snapshot/conversion.go b/core/state/snapshot/conversion.go index f70cbf1e686b1..0f3934cb423bb 100644 --- a/core/state/snapshot/conversion.go +++ b/core/state/snapshot/conversion.go @@ -43,7 +43,7 @@ type trieKV struct { type ( // trieGeneratorFn is the interface of trie generation which can // be implemented by different trie algorithm. - trieGeneratorFn func(db ethdb.KeyValueWriter, in chan (trieKV), out chan (common.Hash)) + trieGeneratorFn func(db ethdb.KeyValueWriter, owner common.Hash, in chan (trieKV), out chan (common.Hash)) // leafCallbackFn is the callback invoked at the leaves of the trie, // returns the subtrie root with the specified subtrie identifier. @@ -253,7 +253,7 @@ func generateTrieRoot(db ethdb.KeyValueWriter, it Iterator, account common.Hash, wg.Add(1) go func() { defer wg.Done() - generatorFn(db, in, out) + generatorFn(db, account, in, out) }() // Spin up a go-routine for progress logging if report && stats != nil { @@ -360,8 +360,8 @@ func generateTrieRoot(db ethdb.KeyValueWriter, it Iterator, account common.Hash, return stop(nil) } -func stackTrieGenerate(db ethdb.KeyValueWriter, in chan trieKV, out chan common.Hash) { - t := trie.NewStackTrie(db) +func stackTrieGenerate(db ethdb.KeyValueWriter, owner common.Hash, in chan trieKV, out chan common.Hash) { + t := trie.NewStackTrieWithOwner(db, owner) for leaf := range in { t.TryUpdate(leaf.key[:], leaf.value) } diff --git a/core/state/snapshot/generate.go b/core/state/snapshot/generate.go index 769989aec21c0..36055856e1c71 100644 --- a/core/state/snapshot/generate.go +++ b/core/state/snapshot/generate.go @@ -166,7 +166,7 @@ func (result *proofResult) forEach(callback func(key []byte, val []byte) error) // // The proof result will be returned if the range proving is finished, otherwise // the error will be returned to abort the entire procedure. -func (dl *diskLayer) proveRange(ctx *generatorContext, root common.Hash, prefix []byte, kind string, origin []byte, max int, valueConvertFn func([]byte) ([]byte, error)) (*proofResult, error) { +func (dl *diskLayer) proveRange(ctx *generatorContext, owner common.Hash, root common.Hash, prefix []byte, kind string, origin []byte, max int, valueConvertFn func([]byte) ([]byte, error)) (*proofResult, error) { var ( keys [][]byte vals [][]byte @@ -234,7 +234,7 @@ func (dl *diskLayer) proveRange(ctx *generatorContext, root common.Hash, prefix // The snap state is exhausted, pass the entire key/val set for verification if origin == nil && !diskMore { - stackTr := trie.NewStackTrie(nil) + stackTr := trie.NewStackTrieWithOwner(nil, owner) for i, key := range keys { stackTr.TryUpdate(key, vals[i]) } @@ -248,7 +248,7 @@ func (dl *diskLayer) proveRange(ctx *generatorContext, root common.Hash, prefix return &proofResult{keys: keys, vals: vals}, nil } // Snap state is chunked, generate edge proofs for verification. - tr, err := trie.New(root, dl.triedb) + tr, err := trie.New(owner, root, dl.triedb) if err != nil { ctx.stats.Log("Trie missing, state snapshotting paused", dl.root, dl.genMarker) return nil, errMissingTrie @@ -313,9 +313,9 @@ type onStateCallback func(key []byte, val []byte, write bool, delete bool) error // generateRange generates the state segment with particular prefix. Generation can // either verify the correctness of existing state through range-proof and skip // generation, or iterate trie to regenerate state on demand. -func (dl *diskLayer) generateRange(ctx *generatorContext, root common.Hash, prefix []byte, kind string, origin []byte, max int, onState onStateCallback, valueConvertFn func([]byte) ([]byte, error)) (bool, []byte, error) { +func (dl *diskLayer) generateRange(ctx *generatorContext, owner common.Hash, root common.Hash, prefix []byte, kind string, origin []byte, max int, onState onStateCallback, valueConvertFn func([]byte) ([]byte, error)) (bool, []byte, error) { // Use range prover to check the validity of the flat state in the range - result, err := dl.proveRange(ctx, root, prefix, kind, origin, max, valueConvertFn) + result, err := dl.proveRange(ctx, owner, root, prefix, kind, origin, max, valueConvertFn) if err != nil { return false, nil, err } @@ -363,7 +363,7 @@ func (dl *diskLayer) generateRange(ctx *generatorContext, root common.Hash, pref if len(result.keys) > 0 { snapNodeCache = memorydb.New() snapTrieDb := trie.NewDatabase(snapNodeCache) - snapTrie, _ := trie.New(common.Hash{}, snapTrieDb) + snapTrie, _ := trie.New(owner, common.Hash{}, snapTrieDb) for i, key := range result.keys { snapTrie.Update(key, result.vals[i]) } @@ -374,7 +374,7 @@ func (dl *diskLayer) generateRange(ctx *generatorContext, root common.Hash, pref // if it's already opened with some nodes resolved. tr := result.tr if tr == nil { - tr, err = trie.New(root, dl.triedb) + tr, err = trie.New(owner, root, dl.triedb) if err != nil { ctx.stats.Log("Trie missing, state snapshotting paused", dl.root, dl.genMarker) return false, nil, errMissingTrie @@ -537,7 +537,7 @@ func generateStorages(ctx *generatorContext, dl *diskLayer, account common.Hash, // Loop for re-generating the missing storage slots. var origin = common.CopyBytes(storeMarker) for { - exhausted, last, err := dl.generateRange(ctx, storageRoot, append(rawdb.SnapshotStoragePrefix, account.Bytes()...), snapStorage, origin, storageCheckRange, onStorage, nil) + exhausted, last, err := dl.generateRange(ctx, account, storageRoot, append(rawdb.SnapshotStoragePrefix, account.Bytes()...), snapStorage, origin, storageCheckRange, onStorage, nil) if err != nil { return err // The procedure it aborted, either by external signal or internal error. } @@ -637,7 +637,7 @@ func generateAccounts(ctx *generatorContext, dl *diskLayer, accMarker []byte) er } origin := common.CopyBytes(accMarker) for { - exhausted, last, err := dl.generateRange(ctx, dl.root, rawdb.SnapshotAccountPrefix, snapAccount, origin, accountRange, onAccount, FullAccountRLP) + exhausted, last, err := dl.generateRange(ctx, common.Hash{}, dl.root, rawdb.SnapshotAccountPrefix, snapAccount, origin, accountRange, onAccount, FullAccountRLP) if err != nil { return err // The procedure it aborted, either by external signal or internal error. } diff --git a/core/state/snapshot/generate_test.go b/core/state/snapshot/generate_test.go index 94caed08ad7a7..fe81993e9d2fd 100644 --- a/core/state/snapshot/generate_test.go +++ b/core/state/snapshot/generate_test.go @@ -26,47 +26,40 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/ethdb/memorydb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" "golang.org/x/crypto/sha3" ) +func hashData(input []byte) common.Hash { + var hasher = sha3.NewLegacyKeccak256() + var hash common.Hash + hasher.Reset() + hasher.Write(input) + hasher.Sum(hash[:0]) + return hash +} + // Tests that snapshot generation from an empty database. func TestGeneration(t *testing.T) { // We can't use statedb to make a test trie (circular dependency), so make // a fake one manually. We're going with a small account trie of 3 accounts, // two of which also has the same 3-slot storage trie attached. - var ( - diskdb = memorydb.New() - triedb = trie.NewDatabase(diskdb) - ) - stTrie, _ := trie.NewSecure(common.Hash{}, triedb) - stTrie.Update([]byte("key-1"), []byte("val-1")) // 0x1314700b81afc49f94db3623ef1df38f3ed18b73a1b7ea2f6c095118cf6118a0 - stTrie.Update([]byte("key-2"), []byte("val-2")) // 0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371 - stTrie.Update([]byte("key-3"), []byte("val-3")) // 0x51c71a47af0695957647fb68766d0becee77e953df17c29b3c2f25436f055c78 - stTrie.Commit(nil) // Root: 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67 - - accTrie, _ := trie.NewSecure(common.Hash{}, triedb) - acc := &Account{Balance: big.NewInt(1), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()} - val, _ := rlp.EncodeToBytes(acc) - accTrie.Update([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e + var helper = newHelper() + stRoot := helper.makeStorageTrie(common.Hash{}, common.Hash{}, []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, false) - acc = &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()} - val, _ = rlp.EncodeToBytes(acc) - accTrie.Update([]byte("acc-2"), val) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7 + helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) + helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) + helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: emptyCode.Bytes()}) - acc = &Account{Balance: big.NewInt(3), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()} - val, _ = rlp.EncodeToBytes(acc) - accTrie.Update([]byte("acc-3"), val) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2 - root, _, _ := accTrie.Commit(nil) // Root: 0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd - triedb.Commit(root, false, nil) + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) + root, snap := helper.CommitAndGenerate() if have, want := root, common.HexToHash("0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd"); have != want { t.Fatalf("have %#x want %#x", have, want) } - snap := generateSnapshot(diskdb, triedb, 16, root) select { case <-snap.genPending: // Snapshot generation succeeded @@ -75,63 +68,34 @@ func TestGeneration(t *testing.T) { t.Errorf("Snapshot generation failed") } checkSnapRoot(t, snap, root) + // Signal abortion to the generator and wait for it to tear down stop := make(chan *generatorStats) snap.genAbort <- stop <-stop } -func hashData(input []byte) common.Hash { - var hasher = sha3.NewLegacyKeccak256() - var hash common.Hash - hasher.Reset() - hasher.Write(input) - hasher.Sum(hash[:0]) - return hash -} - // Tests that snapshot generation with existent flat state. func TestGenerateExistentState(t *testing.T) { // We can't use statedb to make a test trie (circular dependency), so make // a fake one manually. We're going with a small account trie of 3 accounts, // two of which also has the same 3-slot storage trie attached. - var ( - diskdb = memorydb.New() - triedb = trie.NewDatabase(diskdb) - ) - stTrie, _ := trie.NewSecure(common.Hash{}, triedb) - stTrie.Update([]byte("key-1"), []byte("val-1")) // 0x1314700b81afc49f94db3623ef1df38f3ed18b73a1b7ea2f6c095118cf6118a0 - stTrie.Update([]byte("key-2"), []byte("val-2")) // 0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371 - stTrie.Update([]byte("key-3"), []byte("val-3")) // 0x51c71a47af0695957647fb68766d0becee77e953df17c29b3c2f25436f055c78 - stTrie.Commit(nil) // Root: 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67 - - accTrie, _ := trie.NewSecure(common.Hash{}, triedb) - acc := &Account{Balance: big.NewInt(1), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()} - val, _ := rlp.EncodeToBytes(acc) - accTrie.Update([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e - rawdb.WriteAccountSnapshot(diskdb, hashData([]byte("acc-1")), val) - rawdb.WriteStorageSnapshot(diskdb, hashData([]byte("acc-1")), hashData([]byte("key-1")), []byte("val-1")) - rawdb.WriteStorageSnapshot(diskdb, hashData([]byte("acc-1")), hashData([]byte("key-2")), []byte("val-2")) - rawdb.WriteStorageSnapshot(diskdb, hashData([]byte("acc-1")), hashData([]byte("key-3")), []byte("val-3")) - - acc = &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()} - val, _ = rlp.EncodeToBytes(acc) - accTrie.Update([]byte("acc-2"), val) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7 - diskdb.Put(hashData([]byte("acc-2")).Bytes(), val) - rawdb.WriteAccountSnapshot(diskdb, hashData([]byte("acc-2")), val) - - acc = &Account{Balance: big.NewInt(3), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()} - val, _ = rlp.EncodeToBytes(acc) - accTrie.Update([]byte("acc-3"), val) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2 - rawdb.WriteAccountSnapshot(diskdb, hashData([]byte("acc-3")), val) - rawdb.WriteStorageSnapshot(diskdb, hashData([]byte("acc-3")), hashData([]byte("key-1")), []byte("val-1")) - rawdb.WriteStorageSnapshot(diskdb, hashData([]byte("acc-3")), hashData([]byte("key-2")), []byte("val-2")) - rawdb.WriteStorageSnapshot(diskdb, hashData([]byte("acc-3")), hashData([]byte("key-3")), []byte("val-3")) - - root, _, _ := accTrie.Commit(nil) // Root: 0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd - triedb.Commit(root, false, nil) - - snap := generateSnapshot(diskdb, triedb, 16, root) + var helper = newHelper() + + stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) + helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) + helper.addSnapAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) + helper.addSnapStorage("acc-1", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) + + helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) + helper.addSnapAccount("acc-2", &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) + + stRoot = helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) + helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: emptyCode.Bytes()}) + helper.addSnapAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: emptyCode.Bytes()}) + helper.addSnapStorage("acc-3", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) + + root, snap := helper.CommitAndGenerate() select { case <-snap.genPending: // Snapshot generation succeeded @@ -140,6 +104,7 @@ func TestGenerateExistentState(t *testing.T) { t.Errorf("Snapshot generation failed") } checkSnapRoot(t, snap, root) + // Signal abortion to the generator and wait for it to tear down stop := make(chan *generatorStats) snap.genAbort <- stop @@ -163,7 +128,6 @@ func checkSnapRoot(t *testing.T, snap *diskLayer, trieRoot common.Hash) { } return hash, nil }, newGenerateStats(), true) - if err != nil { t.Fatal(err) } @@ -171,20 +135,20 @@ func checkSnapRoot(t *testing.T, snap *diskLayer, trieRoot common.Hash) { t.Fatalf("snaproot: %#x != trieroot #%x", snapRoot, trieRoot) } if err := CheckDanglingStorage(snap.diskdb); err != nil { - t.Fatalf("Detected dangling storages %v", err) + t.Fatalf("Detected dangling storages: %v", err) } } type testHelper struct { - diskdb *memorydb.Database + diskdb ethdb.Database triedb *trie.Database accTrie *trie.SecureTrie } func newHelper() *testHelper { - diskdb := memorydb.New() + diskdb := rawdb.NewMemoryDatabase() triedb := trie.NewDatabase(diskdb) - accTrie, _ := trie.NewSecure(common.Hash{}, triedb) + accTrie, _ := trie.NewSecure(common.Hash{}, common.Hash{}, triedb) return &testHelper{ diskdb: diskdb, triedb: triedb, @@ -215,18 +179,28 @@ func (t *testHelper) addSnapStorage(accKey string, keys []string, vals []string) } } -func (t *testHelper) makeStorageTrie(keys []string, vals []string) []byte { - stTrie, _ := trie.NewSecure(common.Hash{}, t.triedb) +func (t *testHelper) makeStorageTrie(stateRoot, owner common.Hash, keys []string, vals []string, commit bool) []byte { + stTrie, _ := trie.NewSecure(owner, common.Hash{}, t.triedb) for i, k := range keys { stTrie.Update([]byte(k), []byte(vals[i])) } - root, _, _ := stTrie.Commit(nil) + var root common.Hash + if !commit { + root = stTrie.Hash() + } else { + root, _, _ = stTrie.Commit(nil) + } return root.Bytes() } -func (t *testHelper) Generate() (common.Hash, *diskLayer) { +func (t *testHelper) Commit() common.Hash { root, _, _ := t.accTrie.Commit(nil) t.triedb.Commit(root, false, nil) + return root +} + +func (t *testHelper) CommitAndGenerate() (common.Hash, *diskLayer) { + root := t.Commit() snap := generateSnapshot(t.diskdb, t.triedb, 16, root) return root, snap } @@ -249,26 +223,29 @@ func (t *testHelper) Generate() (common.Hash, *diskLayer) { // - extra slots in the end func TestGenerateExistentStateWithWrongStorage(t *testing.T) { helper := newHelper() - stRoot := helper.makeStorageTrie([]string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) // Account one, empty root but non-empty database helper.addAccount("acc-1", &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) helper.addSnapStorage("acc-1", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) // Account two, non empty root but empty database + stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-2")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addAccount("acc-2", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) // Miss slots { // Account three, non empty root but misses slots in the beginning + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addAccount("acc-3", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) helper.addSnapStorage("acc-3", []string{"key-2", "key-3"}, []string{"val-2", "val-3"}) // Account four, non empty root but misses slots in the middle + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-4")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addAccount("acc-4", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) helper.addSnapStorage("acc-4", []string{"key-1", "key-3"}, []string{"val-1", "val-3"}) // Account five, non empty root but misses slots in the end + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-5")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addAccount("acc-5", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) helper.addSnapStorage("acc-5", []string{"key-1", "key-2"}, []string{"val-1", "val-2"}) } @@ -276,18 +253,22 @@ func TestGenerateExistentStateWithWrongStorage(t *testing.T) { // Wrong storage slots { // Account six, non empty root but wrong slots in the beginning + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-6")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addAccount("acc-6", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) helper.addSnapStorage("acc-6", []string{"key-1", "key-2", "key-3"}, []string{"badval-1", "val-2", "val-3"}) // Account seven, non empty root but wrong slots in the middle + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-7")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addAccount("acc-7", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) helper.addSnapStorage("acc-7", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "badval-2", "val-3"}) // Account eight, non empty root but wrong slots in the end + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-8")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addAccount("acc-8", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) helper.addSnapStorage("acc-8", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "badval-3"}) // Account 9, non empty root but rotated slots + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-9")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addAccount("acc-9", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) helper.addSnapStorage("acc-9", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-3", "val-2"}) } @@ -295,19 +276,22 @@ func TestGenerateExistentStateWithWrongStorage(t *testing.T) { // Extra storage slots { // Account 10, non empty root but extra slots in the beginning + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-10")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addAccount("acc-10", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) helper.addSnapStorage("acc-10", []string{"key-0", "key-1", "key-2", "key-3"}, []string{"val-0", "val-1", "val-2", "val-3"}) // Account 11, non empty root but extra slots in the middle + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-11")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addAccount("acc-11", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) helper.addSnapStorage("acc-11", []string{"key-1", "key-2", "key-2-1", "key-3"}, []string{"val-1", "val-2", "val-2-1", "val-3"}) // Account 12, non empty root but extra slots in the end + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-12")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addAccount("acc-12", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) helper.addSnapStorage("acc-12", []string{"key-1", "key-2", "key-3", "key-4"}, []string{"val-1", "val-2", "val-3", "val-4"}) } - root, snap := helper.Generate() + root, snap := helper.CommitAndGenerate() t.Logf("Root: %#x\n", root) // Root = 0x8746cce9fd9c658b2cfd639878ed6584b7a2b3e73bb40f607fcfa156002429a0 select { @@ -331,7 +315,12 @@ func TestGenerateExistentStateWithWrongStorage(t *testing.T) { // - extra accounts func TestGenerateExistentStateWithWrongAccounts(t *testing.T) { helper := newHelper() - stRoot := helper.makeStorageTrie([]string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) + + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-2")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-4")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) + stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-6")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) // Trie accounts [acc-1, acc-2, acc-3, acc-4, acc-6] // Extra accounts [acc-0, acc-5, acc-7] @@ -359,7 +348,7 @@ func TestGenerateExistentStateWithWrongAccounts(t *testing.T) { helper.addSnapAccount("acc-7", &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyRoot.Bytes()}) // after the end } - root, snap := helper.Generate() + root, snap := helper.CommitAndGenerate() t.Logf("Root: %#x\n", root) // Root = 0x825891472281463511e7ebcc7f109e4f9200c20fa384754e11fd605cd98464e8 select { @@ -383,29 +372,19 @@ func TestGenerateCorruptAccountTrie(t *testing.T) { // We can't use statedb to make a test trie (circular dependency), so make // a fake one manually. We're going with a small account trie of 3 accounts, // without any storage slots to keep the test smaller. - var ( - diskdb = memorydb.New() - triedb = trie.NewDatabase(diskdb) - ) - tr, _ := trie.NewSecure(common.Hash{}, triedb) - acc := &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()} - val, _ := rlp.EncodeToBytes(acc) - tr.Update([]byte("acc-1"), val) // 0xc7a30f39aff471c95d8a837497ad0e49b65be475cc0953540f80cfcdbdcd9074 + helper := newHelper() - acc = &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()} - val, _ = rlp.EncodeToBytes(acc) - tr.Update([]byte("acc-2"), val) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7 + helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) // 0xc7a30f39aff471c95d8a837497ad0e49b65be475cc0953540f80cfcdbdcd9074 + helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7 + helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) // 0x19ead688e907b0fab07176120dceec244a72aff2f0aa51e8b827584e378772f4 - acc = &Account{Balance: big.NewInt(3), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()} - val, _ = rlp.EncodeToBytes(acc) - tr.Update([]byte("acc-3"), val) // 0x19ead688e907b0fab07176120dceec244a72aff2f0aa51e8b827584e378772f4 - tr.Commit(nil) // Root: 0xa04693ea110a31037fb5ee814308a6f1d76bdab0b11676bdf4541d2de55ba978 + root, _, _ := helper.accTrie.Commit(nil) // Root: 0xa04693ea110a31037fb5ee814308a6f1d76bdab0b11676bdf4541d2de55ba978 // Delete an account trie leaf and ensure the generator chokes - triedb.Commit(common.HexToHash("0xa04693ea110a31037fb5ee814308a6f1d76bdab0b11676bdf4541d2de55ba978"), false, nil) - diskdb.Delete(common.HexToHash("0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7").Bytes()) + helper.triedb.Commit(root, false, nil) + helper.diskdb.Delete(common.HexToHash("0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7").Bytes()) - snap := generateSnapshot(diskdb, triedb, 16, common.HexToHash("0xa04693ea110a31037fb5ee814308a6f1d76bdab0b11676bdf4541d2de55ba978")) + snap := generateSnapshot(helper.diskdb, helper.triedb, 16, root) select { case <-snap.genPending: // Snapshot generation succeeded @@ -427,45 +406,30 @@ func TestGenerateMissingStorageTrie(t *testing.T) { // We can't use statedb to make a test trie (circular dependency), so make // a fake one manually. We're going with a small account trie of 3 accounts, // two of which also has the same 3-slot storage trie attached. - var ( - diskdb = memorydb.New() - triedb = trie.NewDatabase(diskdb) - ) - stTrie, _ := trie.NewSecure(common.Hash{}, triedb) - stTrie.Update([]byte("key-1"), []byte("val-1")) // 0x1314700b81afc49f94db3623ef1df38f3ed18b73a1b7ea2f6c095118cf6118a0 - stTrie.Update([]byte("key-2"), []byte("val-2")) // 0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371 - stTrie.Update([]byte("key-3"), []byte("val-3")) // 0x51c71a47af0695957647fb68766d0becee77e953df17c29b3c2f25436f055c78 - stTrie.Commit(nil) // Root: 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67 - - accTrie, _ := trie.NewSecure(common.Hash{}, triedb) - acc := &Account{Balance: big.NewInt(1), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()} - val, _ := rlp.EncodeToBytes(acc) - accTrie.Update([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e - - acc = &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()} - val, _ = rlp.EncodeToBytes(acc) - accTrie.Update([]byte("acc-2"), val) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7 + helper := newHelper() - acc = &Account{Balance: big.NewInt(3), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()} - val, _ = rlp.EncodeToBytes(acc) - accTrie.Update([]byte("acc-3"), val) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2 - accTrie.Commit(nil) // Root: 0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd + stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) // 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67 + helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e + helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7 + stRoot = helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) + helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: emptyCode.Bytes()}) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2 + root, _, _ := helper.accTrie.Commit(nil) // We can only corrupt the disk database, so flush the tries out - triedb.Reference( - common.HexToHash("0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67"), + helper.triedb.Reference( + common.BytesToHash(stRoot), common.HexToHash("0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e"), ) - triedb.Reference( - common.HexToHash("0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67"), + helper.triedb.Reference( + common.BytesToHash(stRoot), common.HexToHash("0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2"), ) - triedb.Commit(common.HexToHash("0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd"), false, nil) + helper.triedb.Commit(root, false, nil) // Delete a storage trie root and ensure the generator chokes - diskdb.Delete(common.HexToHash("0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67").Bytes()) + helper.diskdb.Delete(stRoot) - snap := generateSnapshot(diskdb, triedb, 16, common.HexToHash("0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd")) + snap := generateSnapshot(helper.diskdb, helper.triedb, 16, root) select { case <-snap.genPending: // Snapshot generation succeeded @@ -486,45 +450,31 @@ func TestGenerateCorruptStorageTrie(t *testing.T) { // We can't use statedb to make a test trie (circular dependency), so make // a fake one manually. We're going with a small account trie of 3 accounts, // two of which also has the same 3-slot storage trie attached. - var ( - diskdb = memorydb.New() - triedb = trie.NewDatabase(diskdb) - ) - stTrie, _ := trie.NewSecure(common.Hash{}, triedb) - stTrie.Update([]byte("key-1"), []byte("val-1")) // 0x1314700b81afc49f94db3623ef1df38f3ed18b73a1b7ea2f6c095118cf6118a0 - stTrie.Update([]byte("key-2"), []byte("val-2")) // 0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371 - stTrie.Update([]byte("key-3"), []byte("val-3")) // 0x51c71a47af0695957647fb68766d0becee77e953df17c29b3c2f25436f055c78 - stTrie.Commit(nil) // Root: 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67 - - accTrie, _ := trie.NewSecure(common.Hash{}, triedb) - acc := &Account{Balance: big.NewInt(1), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()} - val, _ := rlp.EncodeToBytes(acc) - accTrie.Update([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e + helper := newHelper() - acc = &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()} - val, _ = rlp.EncodeToBytes(acc) - accTrie.Update([]byte("acc-2"), val) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7 + stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) // 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67 + helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e + helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7 + stRoot = helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) + helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: emptyCode.Bytes()}) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2 - acc = &Account{Balance: big.NewInt(3), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()} - val, _ = rlp.EncodeToBytes(acc) - accTrie.Update([]byte("acc-3"), val) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2 - accTrie.Commit(nil) // Root: 0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd + root, _, _ := helper.accTrie.Commit(nil) // We can only corrupt the disk database, so flush the tries out - triedb.Reference( - common.HexToHash("0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67"), + helper.triedb.Reference( + common.BytesToHash(stRoot), common.HexToHash("0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e"), ) - triedb.Reference( - common.HexToHash("0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67"), + helper.triedb.Reference( + common.BytesToHash(stRoot), common.HexToHash("0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2"), ) - triedb.Commit(common.HexToHash("0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd"), false, nil) + helper.triedb.Commit(root, false, nil) // Delete a storage trie leaf and ensure the generator chokes - diskdb.Delete(common.HexToHash("0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371").Bytes()) + helper.diskdb.Delete(common.HexToHash("0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371").Bytes()) - snap := generateSnapshot(diskdb, triedb, 16, common.HexToHash("0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd")) + snap := generateSnapshot(helper.diskdb, helper.triedb, 16, root) select { case <-snap.genPending: // Snapshot generation succeeded @@ -539,56 +489,51 @@ func TestGenerateCorruptStorageTrie(t *testing.T) { <-stop } -func getStorageTrie(n int, triedb *trie.Database) *trie.SecureTrie { - stTrie, _ := trie.NewSecure(common.Hash{}, triedb) - for i := 0; i < n; i++ { - k := fmt.Sprintf("key-%d", i) - v := fmt.Sprintf("val-%d", i) - stTrie.Update([]byte(k), []byte(v)) - } - stTrie.Commit(nil) - return stTrie -} - // Tests that snapshot generation when an extra account with storage exists in the snap state. func TestGenerateWithExtraAccounts(t *testing.T) { - var ( - diskdb = memorydb.New() - triedb = trie.NewDatabase(diskdb) - stTrie = getStorageTrie(5, triedb) - ) - accTrie, _ := trie.NewSecure(common.Hash{}, triedb) - { // Account one in the trie - acc := &Account{Balance: big.NewInt(1), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()} + helper := newHelper() + { + // Account one in the trie + stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), + []string{"key-1", "key-2", "key-3", "key-4", "key-5"}, + []string{"val-1", "val-2", "val-3", "val-4", "val-5"}, + true, + ) + acc := &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()} val, _ := rlp.EncodeToBytes(acc) - accTrie.Update([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e + helper.accTrie.Update([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e + // Identical in the snap key := hashData([]byte("acc-1")) - rawdb.WriteAccountSnapshot(diskdb, key, val) - rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("key-1")), []byte("val-1")) - rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("key-2")), []byte("val-2")) - rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("key-3")), []byte("val-3")) - rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("key-4")), []byte("val-4")) - rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("key-5")), []byte("val-5")) - } - { // Account two exists only in the snapshot - acc := &Account{Balance: big.NewInt(1), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()} + rawdb.WriteAccountSnapshot(helper.triedb.DiskDB(), key, val) + rawdb.WriteStorageSnapshot(helper.triedb.DiskDB(), key, hashData([]byte("key-1")), []byte("val-1")) + rawdb.WriteStorageSnapshot(helper.triedb.DiskDB(), key, hashData([]byte("key-2")), []byte("val-2")) + rawdb.WriteStorageSnapshot(helper.triedb.DiskDB(), key, hashData([]byte("key-3")), []byte("val-3")) + rawdb.WriteStorageSnapshot(helper.triedb.DiskDB(), key, hashData([]byte("key-4")), []byte("val-4")) + rawdb.WriteStorageSnapshot(helper.triedb.DiskDB(), key, hashData([]byte("key-5")), []byte("val-5")) + } + { + // Account two exists only in the snapshot + stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-2")), + []string{"key-1", "key-2", "key-3", "key-4", "key-5"}, + []string{"val-1", "val-2", "val-3", "val-4", "val-5"}, + true, + ) + acc := &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()} val, _ := rlp.EncodeToBytes(acc) key := hashData([]byte("acc-2")) - rawdb.WriteAccountSnapshot(diskdb, key, val) - rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("b-key-1")), []byte("b-val-1")) - rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("b-key-2")), []byte("b-val-2")) - rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("b-key-3")), []byte("b-val-3")) - } - root, _, _ := accTrie.Commit(nil) - t.Logf("root: %x", root) - triedb.Commit(root, false, nil) + rawdb.WriteAccountSnapshot(helper.triedb.DiskDB(), key, val) + rawdb.WriteStorageSnapshot(helper.triedb.DiskDB(), key, hashData([]byte("b-key-1")), []byte("b-val-1")) + rawdb.WriteStorageSnapshot(helper.triedb.DiskDB(), key, hashData([]byte("b-key-2")), []byte("b-val-2")) + rawdb.WriteStorageSnapshot(helper.triedb.DiskDB(), key, hashData([]byte("b-key-3")), []byte("b-val-3")) + } + root := helper.Commit() + // To verify the test: If we now inspect the snap db, there should exist extraneous storage items - if data := rawdb.ReadStorageSnapshot(diskdb, hashData([]byte("acc-2")), hashData([]byte("b-key-1"))); data == nil { + if data := rawdb.ReadStorageSnapshot(helper.triedb.DiskDB(), hashData([]byte("acc-2")), hashData([]byte("b-key-1"))); data == nil { t.Fatalf("expected snap storage to exist") } - - snap := generateSnapshot(diskdb, triedb, 16, root) + snap := generateSnapshot(helper.diskdb, helper.triedb, 16, root) select { case <-snap.genPending: // Snapshot generation succeeded @@ -597,12 +542,13 @@ func TestGenerateWithExtraAccounts(t *testing.T) { t.Errorf("Snapshot generation failed") } checkSnapRoot(t, snap, root) + // Signal abortion to the generator and wait for it to tear down stop := make(chan *generatorStats) snap.genAbort <- stop <-stop // If we now inspect the snap db, there should exist no extraneous storage items - if data := rawdb.ReadStorageSnapshot(diskdb, hashData([]byte("acc-2")), hashData([]byte("b-key-1"))); data != nil { + if data := rawdb.ReadStorageSnapshot(helper.triedb.DiskDB(), hashData([]byte("acc-2")), hashData([]byte("b-key-1"))); data != nil { t.Fatalf("expected slot to be removed, got %v", string(data)) } } @@ -616,37 +562,36 @@ func TestGenerateWithManyExtraAccounts(t *testing.T) { if false { enableLogging() } - var ( - diskdb = memorydb.New() - triedb = trie.NewDatabase(diskdb) - stTrie = getStorageTrie(3, triedb) - ) - accTrie, _ := trie.NewSecure(common.Hash{}, triedb) - { // Account one in the trie - acc := &Account{Balance: big.NewInt(1), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()} + helper := newHelper() + { + // Account one in the trie + stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), + []string{"key-1", "key-2", "key-3"}, + []string{"val-1", "val-2", "val-3"}, + true, + ) + acc := &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()} val, _ := rlp.EncodeToBytes(acc) - accTrie.Update([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e + helper.accTrie.Update([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e + // Identical in the snap key := hashData([]byte("acc-1")) - rawdb.WriteAccountSnapshot(diskdb, key, val) - rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("key-1")), []byte("val-1")) - rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("key-2")), []byte("val-2")) - rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("key-3")), []byte("val-3")) + rawdb.WriteAccountSnapshot(helper.diskdb, key, val) + rawdb.WriteStorageSnapshot(helper.diskdb, key, hashData([]byte("key-1")), []byte("val-1")) + rawdb.WriteStorageSnapshot(helper.diskdb, key, hashData([]byte("key-2")), []byte("val-2")) + rawdb.WriteStorageSnapshot(helper.diskdb, key, hashData([]byte("key-3")), []byte("val-3")) } - { // 100 accounts exist only in snapshot + { + // 100 accounts exist only in snapshot for i := 0; i < 1000; i++ { //acc := &Account{Balance: big.NewInt(int64(i)), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()} acc := &Account{Balance: big.NewInt(int64(i)), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()} val, _ := rlp.EncodeToBytes(acc) key := hashData([]byte(fmt.Sprintf("acc-%d", i))) - rawdb.WriteAccountSnapshot(diskdb, key, val) + rawdb.WriteAccountSnapshot(helper.diskdb, key, val) } } - root, _, _ := accTrie.Commit(nil) - t.Logf("root: %x", root) - triedb.Commit(root, false, nil) - - snap := generateSnapshot(diskdb, triedb, 16, root) + root, snap := helper.CommitAndGenerate() select { case <-snap.genPending: // Snapshot generation succeeded @@ -675,31 +620,22 @@ func TestGenerateWithExtraBeforeAndAfter(t *testing.T) { if false { enableLogging() } - var ( - diskdb = memorydb.New() - triedb = trie.NewDatabase(diskdb) - ) - accTrie, _ := trie.New(common.Hash{}, triedb) + helper := newHelper() { acc := &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()} val, _ := rlp.EncodeToBytes(acc) - accTrie.Update(common.HexToHash("0x03").Bytes(), val) - accTrie.Update(common.HexToHash("0x07").Bytes(), val) - - rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x01"), val) - rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x02"), val) - rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x03"), val) - rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x04"), val) - rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x05"), val) - rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x06"), val) - rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x07"), val) - } - - root, _, _ := accTrie.Commit(nil) - t.Logf("root: %x", root) - triedb.Commit(root, false, nil) - - snap := generateSnapshot(diskdb, triedb, 16, root) + helper.accTrie.Update(common.HexToHash("0x03").Bytes(), val) + helper.accTrie.Update(common.HexToHash("0x07").Bytes(), val) + + rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x01"), val) + rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x02"), val) + rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x03"), val) + rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x04"), val) + rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x05"), val) + rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x06"), val) + rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x07"), val) + } + root, snap := helper.CommitAndGenerate() select { case <-snap.genPending: // Snapshot generation succeeded @@ -721,29 +657,20 @@ func TestGenerateWithMalformedSnapdata(t *testing.T) { if false { enableLogging() } - var ( - diskdb = memorydb.New() - triedb = trie.NewDatabase(diskdb) - ) - accTrie, _ := trie.New(common.Hash{}, triedb) + helper := newHelper() { acc := &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()} val, _ := rlp.EncodeToBytes(acc) - accTrie.Update(common.HexToHash("0x03").Bytes(), val) + helper.accTrie.Update(common.HexToHash("0x03").Bytes(), val) junk := make([]byte, 100) copy(junk, []byte{0xde, 0xad}) - rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x02"), junk) - rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x03"), junk) - rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x04"), junk) - rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x05"), junk) + rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x02"), junk) + rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x03"), junk) + rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x04"), junk) + rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x05"), junk) } - - root, _, _ := accTrie.Commit(nil) - t.Logf("root: %x", root) - triedb.Commit(root, false, nil) - - snap := generateSnapshot(diskdb, triedb, 16, root) + root, snap := helper.CommitAndGenerate() select { case <-snap.genPending: // Snapshot generation succeeded @@ -757,7 +684,7 @@ func TestGenerateWithMalformedSnapdata(t *testing.T) { snap.genAbort <- stop <-stop // If we now inspect the snap db, there should exist no extraneous storage items - if data := rawdb.ReadStorageSnapshot(diskdb, hashData([]byte("acc-2")), hashData([]byte("b-key-1"))); data != nil { + if data := rawdb.ReadStorageSnapshot(helper.diskdb, hashData([]byte("acc-2")), hashData([]byte("b-key-1"))); data != nil { t.Fatalf("expected slot to be removed, got %v", string(data)) } } @@ -767,13 +694,13 @@ func TestGenerateFromEmptySnap(t *testing.T) { accountCheckRange = 10 storageCheckRange = 20 helper := newHelper() - stRoot := helper.makeStorageTrie([]string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) // Add 1K accounts to the trie for i := 0; i < 400; i++ { + stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte(fmt.Sprintf("acc-%d", i))), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addTrieAccount(fmt.Sprintf("acc-%d", i), &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) } - root, snap := helper.Generate() + root, snap := helper.CommitAndGenerate() t.Logf("Root: %#x\n", root) // Root: 0x6f7af6d2e1a1bf2b84a3beb3f8b64388465fbc1e274ca5d5d3fc787ca78f59e4 select { @@ -802,12 +729,12 @@ func TestGenerateWithIncompleteStorage(t *testing.T) { helper := newHelper() stKeys := []string{"1", "2", "3", "4", "5", "6", "7", "8"} stVals := []string{"v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8"} - stRoot := helper.makeStorageTrie(stKeys, stVals) // We add 8 accounts, each one is missing exactly one of the storage slots. This means // we don't have to order the keys and figure out exactly which hash-key winds up // on the sensitive spots at the boundaries for i := 0; i < 8; i++ { accKey := fmt.Sprintf("acc-%d", i) + stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte(accKey)), stKeys, stVals, true) helper.addAccount(accKey, &Account{Balance: big.NewInt(int64(i)), Root: stRoot, CodeHash: emptyCode.Bytes()}) var moddedKeys []string var moddedVals []string @@ -819,8 +746,7 @@ func TestGenerateWithIncompleteStorage(t *testing.T) { } helper.addSnapStorage(accKey, moddedKeys, moddedVals) } - - root, snap := helper.Generate() + root, snap := helper.CommitAndGenerate() t.Logf("Root: %#x\n", root) // Root: 0xca73f6f05ba4ca3024ef340ef3dfca8fdabc1b677ff13f5a9571fd49c16e67ff select { @@ -899,7 +825,7 @@ func populateDangling(disk ethdb.KeyValueStore) { // This test will populate some dangling storages to see if they can be cleaned up. func TestGenerateCompleteSnapshotWithDanglingStorage(t *testing.T) { var helper = newHelper() - stRoot := helper.makeStorageTrie([]string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) + stRoot := helper.makeStorageTrie(common.Hash{}, common.Hash{}, []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) helper.addAccount("acc-2", &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) @@ -910,7 +836,7 @@ func TestGenerateCompleteSnapshotWithDanglingStorage(t *testing.T) { populateDangling(helper.diskdb) - root, snap := helper.Generate() + root, snap := helper.CommitAndGenerate() select { case <-snap.genPending: // Snapshot generation succeeded @@ -932,7 +858,7 @@ func TestGenerateCompleteSnapshotWithDanglingStorage(t *testing.T) { // This test will populate some dangling storages to see if they can be cleaned up. func TestGenerateBrokenSnapshotWithDanglingStorage(t *testing.T) { var helper = newHelper() - stRoot := helper.makeStorageTrie([]string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) + stRoot := helper.makeStorageTrie(common.Hash{}, common.Hash{}, []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) @@ -940,7 +866,7 @@ func TestGenerateBrokenSnapshotWithDanglingStorage(t *testing.T) { populateDangling(helper.diskdb) - root, snap := helper.Generate() + root, snap := helper.CommitAndGenerate() select { case <-snap.genPending: // Snapshot generation succeeded diff --git a/core/state/snapshot/journal.go b/core/state/snapshot/journal.go index 6836a574090cd..9a22f27373068 100644 --- a/core/state/snapshot/journal.go +++ b/core/state/snapshot/journal.go @@ -108,44 +108,15 @@ func loadAndParseJournal(db ethdb.KeyValueStore, base *diskLayer) (snapshot, jou // So if there is no journal, or the journal is invalid(e.g. the journal // is not matched with disk layer; or the it's the legacy-format journal, // etc.), we just discard all diffs and try to recover them later. - journal := rawdb.ReadSnapshotJournal(db) - if len(journal) == 0 { - log.Warn("Loaded snapshot journal", "diskroot", base.root, "diffs", "missing") - return base, generator, nil - } - r := rlp.NewStream(bytes.NewReader(journal), 0) - - // Firstly, resolve the first element as the journal version - version, err := r.Uint() + var current snapshot = base + err := iterateJournal(db, func(parent common.Hash, root common.Hash, destructSet map[common.Hash]struct{}, accountData map[common.Hash][]byte, storageData map[common.Hash]map[common.Hash][]byte) error { + current = newDiffLayer(current, root, destructSet, accountData, storageData) + return nil + }) if err != nil { - log.Warn("Failed to resolve the journal version", "error", err) - return base, generator, nil - } - if version != journalVersion { - log.Warn("Discarded the snapshot journal with wrong version", "required", journalVersion, "got", version) - return base, generator, nil - } - // Secondly, resolve the disk layer root, ensure it's continuous - // with disk layer. Note now we can ensure it's the snapshot journal - // correct version, so we expect everything can be resolved properly. - var root common.Hash - if err := r.Decode(&root); err != nil { - return nil, journalGenerator{}, errors.New("missing disk layer root") - } - // The diff journal is not matched with disk, discard them. - // It can happen that Geth crashes without persisting the latest - // diff journal. - if !bytes.Equal(root.Bytes(), base.root.Bytes()) { - log.Warn("Loaded snapshot journal", "diskroot", base.root, "diffs", "unmatched") return base, generator, nil } - // Load all the snapshot diffs from the journal - snapshot, err := loadDiffLayer(base, r) - if err != nil { - return nil, journalGenerator{}, err - } - log.Debug("Loaded snapshot journal", "diskroot", base.root, "diffhead", snapshot.Root()) - return snapshot, generator, nil + return current, generator, nil } // loadSnapshot loads a pre-existing state snapshot backed by a key-value store. @@ -218,57 +189,6 @@ func loadSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, return snapshot, false, nil } -// loadDiffLayer reads the next sections of a snapshot journal, reconstructing a new -// diff and verifying that it can be linked to the requested parent. -func loadDiffLayer(parent snapshot, r *rlp.Stream) (snapshot, error) { - // Read the next diff journal entry - var root common.Hash - if err := r.Decode(&root); err != nil { - // The first read may fail with EOF, marking the end of the journal - if err == io.EOF { - return parent, nil - } - return nil, fmt.Errorf("load diff root: %v", err) - } - var destructs []journalDestruct - if err := r.Decode(&destructs); err != nil { - return nil, fmt.Errorf("load diff destructs: %v", err) - } - destructSet := make(map[common.Hash]struct{}) - for _, entry := range destructs { - destructSet[entry.Hash] = struct{}{} - } - var accounts []journalAccount - if err := r.Decode(&accounts); err != nil { - return nil, fmt.Errorf("load diff accounts: %v", err) - } - accountData := make(map[common.Hash][]byte) - for _, entry := range accounts { - if len(entry.Blob) > 0 { // RLP loses nil-ness, but `[]byte{}` is not a valid item, so reinterpret that - accountData[entry.Hash] = entry.Blob - } else { - accountData[entry.Hash] = nil - } - } - var storage []journalStorage - if err := r.Decode(&storage); err != nil { - return nil, fmt.Errorf("load diff storage: %v", err) - } - storageData := make(map[common.Hash]map[common.Hash][]byte) - for _, entry := range storage { - slots := make(map[common.Hash][]byte) - for i, key := range entry.Keys { - if len(entry.Vals[i]) > 0 { // RLP loses nil-ness, but `[]byte{}` is not a valid item, so reinterpret that - slots[key] = entry.Vals[i] - } else { - slots[key] = nil - } - } - storageData[entry.Hash] = slots - } - return loadDiffLayer(newDiffLayer(parent, root, destructSet, accountData, storageData), r) -} - // Journal terminates any in-progress snapshot generation, also implicitly pushing // the progress into the database. func (dl *diskLayer) Journal(buffer *bytes.Buffer) (common.Hash, error) { @@ -345,3 +265,96 @@ func (dl *diffLayer) Journal(buffer *bytes.Buffer) (common.Hash, error) { log.Debug("Journalled diff layer", "root", dl.root, "parent", dl.parent.Root()) return base, nil } + +// journalCallback is a function which is invoked by iterateJournal, every +// time a difflayer is loaded from disk. +type journalCallback = func(parent common.Hash, root common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) error + +// iterateJournal iterates through the journalled difflayers, loading them from +// the database, and invoking the callback for each loaded layer. +// The order is incremental; starting with the bottom-most difflayer, going towards +// the most recent layer. +// This method returns error either if there was some error reading from disk, +// OR if the callback returns an error when invoked. +func iterateJournal(db ethdb.KeyValueReader, callback journalCallback) error { + journal := rawdb.ReadSnapshotJournal(db) + if len(journal) == 0 { + log.Warn("Loaded snapshot journal", "diffs", "missing") + return nil + } + r := rlp.NewStream(bytes.NewReader(journal), 0) + // Firstly, resolve the first element as the journal version + version, err := r.Uint64() + if err != nil { + log.Warn("Failed to resolve the journal version", "error", err) + return errors.New("failed to resolve journal version") + } + if version != journalVersion { + log.Warn("Discarded the snapshot journal with wrong version", "required", journalVersion, "got", version) + return errors.New("wrong journal version") + } + // Secondly, resolve the disk layer root, ensure it's continuous + // with disk layer. Note now we can ensure it's the snapshot journal + // correct version, so we expect everything can be resolved properly. + var parent common.Hash + if err := r.Decode(&parent); err != nil { + return errors.New("missing disk layer root") + } + if baseRoot := rawdb.ReadSnapshotRoot(db); baseRoot != parent { + log.Warn("Loaded snapshot journal", "diskroot", baseRoot, "diffs", "unmatched") + return fmt.Errorf("mismatched disk and diff layers") + } + for { + var ( + root common.Hash + destructs []journalDestruct + accounts []journalAccount + storage []journalStorage + destructSet = make(map[common.Hash]struct{}) + accountData = make(map[common.Hash][]byte) + storageData = make(map[common.Hash]map[common.Hash][]byte) + ) + // Read the next diff journal entry + if err := r.Decode(&root); err != nil { + // The first read may fail with EOF, marking the end of the journal + if errors.Is(err, io.EOF) { + return nil + } + return fmt.Errorf("load diff root: %v", err) + } + if err := r.Decode(&destructs); err != nil { + return fmt.Errorf("load diff destructs: %v", err) + } + if err := r.Decode(&accounts); err != nil { + return fmt.Errorf("load diff accounts: %v", err) + } + if err := r.Decode(&storage); err != nil { + return fmt.Errorf("load diff storage: %v", err) + } + for _, entry := range destructs { + destructSet[entry.Hash] = struct{}{} + } + for _, entry := range accounts { + if len(entry.Blob) > 0 { // RLP loses nil-ness, but `[]byte{}` is not a valid item, so reinterpret that + accountData[entry.Hash] = entry.Blob + } else { + accountData[entry.Hash] = nil + } + } + for _, entry := range storage { + slots := make(map[common.Hash][]byte) + for i, key := range entry.Keys { + if len(entry.Vals[i]) > 0 { // RLP loses nil-ness, but `[]byte{}` is not a valid item, so reinterpret that + slots[key] = entry.Vals[i] + } else { + slots[key] = nil + } + } + storageData[entry.Hash] = slots + } + if err := callback(parent, root, destructSet, accountData, storageData); err != nil { + return err + } + parent = root + } +} diff --git a/core/state/snapshot/dangling.go b/core/state/snapshot/utils.go similarity index 55% rename from core/state/snapshot/dangling.go rename to core/state/snapshot/utils.go index ca73da793f7a4..fa1f216e6826f 100644 --- a/core/state/snapshot/dangling.go +++ b/core/state/snapshot/utils.go @@ -18,9 +18,7 @@ package snapshot import ( "bytes" - "errors" "fmt" - "io" "time" "github.com/ethereum/go-ethereum/common" @@ -34,7 +32,7 @@ import ( // storage also has corresponding account data. func CheckDanglingStorage(chaindb ethdb.KeyValueStore) error { if err := checkDanglingDiskStorage(chaindb); err != nil { - return err + log.Error("Database check error", "err", err) } return checkDanglingMemStorage(chaindb) } @@ -75,81 +73,80 @@ func checkDanglingDiskStorage(chaindb ethdb.KeyValueStore) error { // checkDanglingMemStorage checks if there is any 'dangling' storage in the journalled // snapshot difflayers. func checkDanglingMemStorage(db ethdb.KeyValueStore) error { - var ( - start = time.Now() - journal = rawdb.ReadSnapshotJournal(db) - ) - if len(journal) == 0 { - log.Warn("Loaded snapshot journal", "diffs", "missing") + start := time.Now() + log.Info("Checking dangling journalled storage") + err := iterateJournal(db, func(pRoot, root common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) error { + for accHash := range storage { + if _, ok := accounts[accHash]; !ok { + log.Error("Dangling storage - missing account", "account", fmt.Sprintf("%#x", accHash), "root", root) + } + } return nil - } - r := rlp.NewStream(bytes.NewReader(journal), 0) - // Firstly, resolve the first element as the journal version - version, err := r.Uint() + }) if err != nil { - log.Warn("Failed to resolve the journal version", "error", err) - return nil - } - if version != journalVersion { - log.Warn("Discarded the snapshot journal with wrong version", "required", journalVersion, "got", version) - return nil - } - // Secondly, resolve the disk layer root, ensure it's continuous - // with disk layer. Note now we can ensure it's the snapshot journal - // correct version, so we expect everything can be resolved properly. - var root common.Hash - if err := r.Decode(&root); err != nil { - return errors.New("missing disk layer root") - } - // The diff journal is not matched with disk, discard them. - // It can happen that Geth crashes without persisting the latest - // diff journal. - // Load all the snapshot diffs from the journal - if err := checkDanglingJournalStorage(r); err != nil { + log.Info("Failed to resolve snapshot journal", "err", err) return err } log.Info("Verified the snapshot journalled storage", "time", common.PrettyDuration(time.Since(start))) return nil } -// loadDiffLayer reads the next sections of a snapshot journal, reconstructing a new -// diff and verifying that it can be linked to the requested parent. -func checkDanglingJournalStorage(r *rlp.Stream) error { - for { - // Read the next diff journal entry - var root common.Hash - if err := r.Decode(&root); err != nil { - // The first read may fail with EOF, marking the end of the journal - if err == io.EOF { - return nil - } - return fmt.Errorf("load diff root: %v", err) +// CheckJournalAccount shows information about an account, from the disk layer and +// up through the diff layers. +func CheckJournalAccount(db ethdb.KeyValueStore, hash common.Hash) error { + // Look up the disk layer first + baseRoot := rawdb.ReadSnapshotRoot(db) + fmt.Printf("Disklayer: Root: %x\n", baseRoot) + if data := rawdb.ReadAccountSnapshot(db, hash); data != nil { + account := new(Account) + if err := rlp.DecodeBytes(data, account); err != nil { + panic(err) } - var destructs []journalDestruct - if err := r.Decode(&destructs); err != nil { - return fmt.Errorf("load diff destructs: %v", err) + fmt.Printf("\taccount.nonce: %d\n", account.Nonce) + fmt.Printf("\taccount.balance: %x\n", account.Balance) + fmt.Printf("\taccount.root: %x\n", account.Root) + fmt.Printf("\taccount.codehash: %x\n", account.CodeHash) + } + // Check storage + { + it := rawdb.NewKeyLengthIterator(db.NewIterator(append(rawdb.SnapshotStoragePrefix, hash.Bytes()...), nil), 1+2*common.HashLength) + fmt.Printf("\tStorage:\n") + for it.Next() { + slot := it.Key()[33:] + fmt.Printf("\t\t%x: %x\n", slot, it.Value()) } - var accounts []journalAccount - if err := r.Decode(&accounts); err != nil { - return fmt.Errorf("load diff accounts: %v", err) + it.Release() + } + var depth = 0 + + return iterateJournal(db, func(pRoot, root common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) error { + _, a := accounts[hash] + _, b := destructs[hash] + _, c := storage[hash] + depth++ + if !a && !b && !c { + return nil } - accountData := make(map[common.Hash][]byte) - for _, entry := range accounts { - if len(entry.Blob) > 0 { // RLP loses nil-ness, but `[]byte{}` is not a valid item, so reinterpret that - accountData[entry.Hash] = entry.Blob - } else { - accountData[entry.Hash] = nil + fmt.Printf("Disklayer+%d: Root: %x, parent %x\n", depth, root, pRoot) + if data, ok := accounts[hash]; ok { + account := new(Account) + if err := rlp.DecodeBytes(data, account); err != nil { + panic(err) } + fmt.Printf("\taccount.nonce: %d\n", account.Nonce) + fmt.Printf("\taccount.balance: %x\n", account.Balance) + fmt.Printf("\taccount.root: %x\n", account.Root) + fmt.Printf("\taccount.codehash: %x\n", account.CodeHash) } - var storage []journalStorage - if err := r.Decode(&storage); err != nil { - return fmt.Errorf("load diff storage: %v", err) + if _, ok := destructs[hash]; ok { + fmt.Printf("\t Destructed!") } - for _, entry := range storage { - if _, ok := accountData[entry.Hash]; !ok { - log.Error("Dangling storage - missing account", "account", fmt.Sprintf("%#x", entry.Hash), "root", root) - return fmt.Errorf("dangling journal snapshot storage account %#x", entry.Hash) + if data, ok := storage[hash]; ok { + fmt.Printf("\tStorage\n") + for k, v := range data { + fmt.Printf("\t\t%x: %x\n", k, v) } } - } + return nil + }) } diff --git a/core/state/state_object.go b/core/state/state_object.go index bcb6dca4f56bc..1ffb7eb402281 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -154,7 +154,7 @@ func (s *stateObject) getTrie(db Database) Trie { if s.data.Root != emptyRoot && s.db.prefetcher != nil { // When the miner is creating the pending state, there is no // prefetcher - s.trie = s.db.prefetcher.trie(s.data.Root) + s.trie = s.db.prefetcher.trie(s.addrHash, s.data.Root) } if s.trie == nil { var err error @@ -295,7 +295,7 @@ func (s *stateObject) finalise(prefetch bool) { } } if s.db.prefetcher != nil && prefetch && len(slotsToPrefetch) > 0 && s.data.Root != emptyRoot { - s.db.prefetcher.prefetch(s.data.Root, slotsToPrefetch) + s.db.prefetcher.prefetch(s.addrHash, s.data.Root, slotsToPrefetch) } if len(s.dirtyStorage) > 0 { s.dirtyStorage = make(Storage) @@ -352,7 +352,7 @@ func (s *stateObject) updateTrie(db Database) Trie { usedStorage = append(usedStorage, common.CopyBytes(key[:])) // Copy needed for closure } if s.db.prefetcher != nil { - s.db.prefetcher.used(s.data.Root, usedStorage) + s.db.prefetcher.used(s.addrHash, s.data.Root, usedStorage) } if len(s.pendingStorage) > 0 { s.pendingStorage = make(Storage) diff --git a/core/state/statedb.go b/core/state/statedb.go index 1d31cf470be03..a36d65fce791b 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -62,11 +62,14 @@ func (n *proofList) Delete(key []byte) error { // * Contracts // * Accounts type StateDB struct { - db Database - prefetcher *triePrefetcher - originalRoot common.Hash // The pre-state root, before any changes were made - trie Trie - hasher crypto.KeccakState + db Database + prefetcher *triePrefetcher + trie Trie + hasher crypto.KeccakState + + // originalRoot is the pre-state root, before any changes were made. + // It will be updated when the Commit is called. + originalRoot common.Hash snaps *snapshot.Tree snap snapshot.Snapshot @@ -648,6 +651,7 @@ func (s *StateDB) Copy() *StateDB { state := &StateDB{ db: s.db, trie: s.db.CopyTrie(s.trie), + originalRoot: s.originalRoot, stateObjects: make(map[common.Address]*stateObject, len(s.journal.dirties)), stateObjectsPending: make(map[common.Address]struct{}, len(s.stateObjectsPending)), stateObjectsDirty: make(map[common.Address]struct{}, len(s.journal.dirties)), @@ -810,7 +814,7 @@ func (s *StateDB) Finalise(deleteEmptyObjects bool) { addressesToPrefetch = append(addressesToPrefetch, common.CopyBytes(addr[:])) // Copy needed for closure } if s.prefetcher != nil && len(addressesToPrefetch) > 0 { - s.prefetcher.prefetch(s.originalRoot, addressesToPrefetch) + s.prefetcher.prefetch(common.Hash{}, s.originalRoot, addressesToPrefetch) } // Invalidate journal because reverting across transactions is not allowed. s.clearJournalAndRefund() @@ -851,7 +855,7 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash { // _untouched_. We can check with the prefetcher, if it can give us a trie // which has the same root, but also has some content loaded into it. if prefetcher != nil { - if trie := prefetcher.trie(s.originalRoot); trie != nil { + if trie := prefetcher.trie(common.Hash{}, s.originalRoot); trie != nil { s.trie = trie } } @@ -867,7 +871,7 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash { usedAddrs = append(usedAddrs, common.CopyBytes(addr[:])) // Copy needed for closure } if prefetcher != nil { - prefetcher.used(s.originalRoot, usedAddrs) + prefetcher.used(common.Hash{}, s.originalRoot, usedAddrs) } if len(s.stateObjectsPending) > 0 { s.stateObjectsPending = make(map[common.Address]struct{}) @@ -980,6 +984,7 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { } s.snap, s.snapDestructs, s.snapAccounts, s.snapStorage = nil, nil, nil, nil } + s.originalRoot = root return root, err } diff --git a/core/state/sync_test.go b/core/state/sync_test.go index 007590c76d9c7..83c5aa2df7a85 100644 --- a/core/state/sync_test.go +++ b/core/state/sync_test.go @@ -104,7 +104,7 @@ func checkTrieConsistency(db ethdb.Database, root common.Hash) error { if v, _ := db.Get(root[:]); v == nil { return nil // Consider a non existent state consistent. } - trie, err := trie.New(root, trie.NewDatabase(db)) + trie, err := trie.New(common.Hash{}, root, trie.NewDatabase(db)) if err != nil { return err } @@ -166,7 +166,7 @@ func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool) { if commit { srcDb.TrieDB().Commit(srcRoot, false, nil) } - srcTrie, _ := trie.New(srcRoot, srcDb.TrieDB()) + srcTrie, _ := trie.New(common.Hash{}, srcRoot, srcDb.TrieDB()) // Create a destination state and sync with the scheduler dstDb := rawdb.NewMemoryDatabase() @@ -207,7 +207,7 @@ func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool) { if err := rlp.DecodeBytes(srcTrie.Get(path[0]), &acc); err != nil { t.Fatalf("failed to decode account on path %x: %v", path, err) } - stTrie, err := trie.New(acc.Root, srcDb.TrieDB()) + stTrie, err := trie.New(common.BytesToHash(path[0]), acc.Root, srcDb.TrieDB()) if err != nil { t.Fatalf("failed to retriev storage trie for path %x: %v", path, err) } diff --git a/core/state/trie_prefetcher.go b/core/state/trie_prefetcher.go index 25c3730e3f7af..4c817b1bc6fbe 100644 --- a/core/state/trie_prefetcher.go +++ b/core/state/trie_prefetcher.go @@ -25,7 +25,7 @@ import ( ) var ( - // triePrefetchMetricsPrefix is the prefix under which to publis the metrics. + // triePrefetchMetricsPrefix is the prefix under which to publish the metrics. triePrefetchMetricsPrefix = "trie/prefetch/" ) @@ -35,10 +35,10 @@ var ( // // Note, the prefetcher's API is not thread safe. type triePrefetcher struct { - db Database // Database to fetch trie nodes through - root common.Hash // Root hash of theaccount trie for metrics - fetches map[common.Hash]Trie // Partially or fully fetcher tries - fetchers map[common.Hash]*subfetcher // Subfetchers for each trie + db Database // Database to fetch trie nodes through + root common.Hash // Root hash of the account trie for metrics + fetches map[string]Trie // Partially or fully fetcher tries + fetchers map[string]*subfetcher // Subfetchers for each trie deliveryMissMeter metrics.Meter accountLoadMeter metrics.Meter @@ -51,13 +51,12 @@ type triePrefetcher struct { storageWasteMeter metrics.Meter } -// newTriePrefetcher func newTriePrefetcher(db Database, root common.Hash, namespace string) *triePrefetcher { prefix := triePrefetchMetricsPrefix + namespace p := &triePrefetcher{ db: db, root: root, - fetchers: make(map[common.Hash]*subfetcher), // Active prefetchers use the fetchers map + fetchers: make(map[string]*subfetcher), // Active prefetchers use the fetchers map deliveryMissMeter: metrics.GetOrRegisterMeter(prefix+"/deliverymiss", nil), accountLoadMeter: metrics.GetOrRegisterMeter(prefix+"/account/load", nil), @@ -112,7 +111,7 @@ func (p *triePrefetcher) copy() *triePrefetcher { copy := &triePrefetcher{ db: p.db, root: p.root, - fetches: make(map[common.Hash]Trie), // Active prefetchers use the fetches map + fetches: make(map[string]Trie), // Active prefetchers use the fetches map deliveryMissMeter: p.deliveryMissMeter, accountLoadMeter: p.accountLoadMeter, @@ -132,33 +131,35 @@ func (p *triePrefetcher) copy() *triePrefetcher { return copy } // Otherwise we're copying an active fetcher, retrieve the current states - for root, fetcher := range p.fetchers { - copy.fetches[root] = fetcher.peek() + for id, fetcher := range p.fetchers { + copy.fetches[id] = fetcher.peek() } return copy } // prefetch schedules a batch of trie items to prefetch. -func (p *triePrefetcher) prefetch(root common.Hash, keys [][]byte) { +func (p *triePrefetcher) prefetch(owner common.Hash, root common.Hash, keys [][]byte) { // If the prefetcher is an inactive one, bail out if p.fetches != nil { return } // Active fetcher, schedule the retrievals - fetcher := p.fetchers[root] + id := p.trieID(owner, root) + fetcher := p.fetchers[id] if fetcher == nil { - fetcher = newSubfetcher(p.db, root) - p.fetchers[root] = fetcher + fetcher = newSubfetcher(p.db, owner, root) + p.fetchers[id] = fetcher } fetcher.schedule(keys) } // trie returns the trie matching the root hash, or nil if the prefetcher doesn't // have it. -func (p *triePrefetcher) trie(root common.Hash) Trie { +func (p *triePrefetcher) trie(owner common.Hash, root common.Hash) Trie { // If the prefetcher is inactive, return from existing deep copies + id := p.trieID(owner, root) if p.fetches != nil { - trie := p.fetches[root] + trie := p.fetches[id] if trie == nil { p.deliveryMissMeter.Mark(1) return nil @@ -166,7 +167,7 @@ func (p *triePrefetcher) trie(root common.Hash) Trie { return p.db.CopyTrie(trie) } // Otherwise the prefetcher is active, bail if no trie was prefetched for this root - fetcher := p.fetchers[root] + fetcher := p.fetchers[id] if fetcher == nil { p.deliveryMissMeter.Mark(1) return nil @@ -185,20 +186,26 @@ func (p *triePrefetcher) trie(root common.Hash) Trie { // used marks a batch of state items used to allow creating statistics as to // how useful or wasteful the prefetcher is. -func (p *triePrefetcher) used(root common.Hash, used [][]byte) { - if fetcher := p.fetchers[root]; fetcher != nil { +func (p *triePrefetcher) used(owner common.Hash, root common.Hash, used [][]byte) { + if fetcher := p.fetchers[p.trieID(owner, root)]; fetcher != nil { fetcher.used = used } } +// trieID returns an unique trie identifier consists the trie owner and root hash. +func (p *triePrefetcher) trieID(owner common.Hash, root common.Hash) string { + return string(append(owner.Bytes(), root.Bytes()...)) +} + // subfetcher is a trie fetcher goroutine responsible for pulling entries for a // single trie. It is spawned when a new root is encountered and lives until the // main prefetcher is paused and either all requested items are processed or if // the trie being worked on is retrieved from the prefetcher. type subfetcher struct { - db Database // Database to load trie nodes through - root common.Hash // Root hash of the trie to prefetch - trie Trie // Trie being populated with nodes + db Database // Database to load trie nodes through + owner common.Hash // Owner of the trie, usually account hash + root common.Hash // Root hash of the trie to prefetch + trie Trie // Trie being populated with nodes tasks [][]byte // Items queued up for retrieval lock sync.Mutex // Lock protecting the task queue @@ -215,15 +222,16 @@ type subfetcher struct { // newSubfetcher creates a goroutine to prefetch state items belonging to a // particular root hash. -func newSubfetcher(db Database, root common.Hash) *subfetcher { +func newSubfetcher(db Database, owner common.Hash, root common.Hash) *subfetcher { sf := &subfetcher{ - db: db, - root: root, - wake: make(chan struct{}, 1), - stop: make(chan struct{}), - term: make(chan struct{}), - copy: make(chan chan Trie), - seen: make(map[string]struct{}), + db: db, + owner: owner, + root: root, + wake: make(chan struct{}, 1), + stop: make(chan struct{}), + term: make(chan struct{}), + copy: make(chan chan Trie), + seen: make(map[string]struct{}), } go sf.loop() return sf @@ -279,13 +287,21 @@ func (sf *subfetcher) loop() { defer close(sf.term) // Start by opening the trie and stop processing if it fails - trie, err := sf.db.OpenTrie(sf.root) - if err != nil { - log.Warn("Trie prefetcher failed opening trie", "root", sf.root, "err", err) - return + if sf.owner == (common.Hash{}) { + trie, err := sf.db.OpenTrie(sf.root) + if err != nil { + log.Warn("Trie prefetcher failed opening trie", "root", sf.root, "err", err) + return + } + sf.trie = trie + } else { + trie, err := sf.db.OpenStorageTrie(sf.owner, sf.root) + if err != nil { + log.Warn("Trie prefetcher failed opening trie", "root", sf.root, "err", err) + return + } + sf.trie = trie } - sf.trie = trie - // Trie opened successfully, keep prefetching items for { select { diff --git a/core/state/trie_prefetcher_test.go b/core/state/trie_prefetcher_test.go index 35dc7a2c0da47..cb0b67d7ea79f 100644 --- a/core/state/trie_prefetcher_test.go +++ b/core/state/trie_prefetcher_test.go @@ -47,20 +47,20 @@ func TestCopyAndClose(t *testing.T) { db := filledStateDB() prefetcher := newTriePrefetcher(db.db, db.originalRoot, "") skey := common.HexToHash("aaa") - prefetcher.prefetch(db.originalRoot, [][]byte{skey.Bytes()}) - prefetcher.prefetch(db.originalRoot, [][]byte{skey.Bytes()}) + prefetcher.prefetch(common.Hash{}, db.originalRoot, [][]byte{skey.Bytes()}) + prefetcher.prefetch(common.Hash{}, db.originalRoot, [][]byte{skey.Bytes()}) time.Sleep(1 * time.Second) - a := prefetcher.trie(db.originalRoot) - prefetcher.prefetch(db.originalRoot, [][]byte{skey.Bytes()}) - b := prefetcher.trie(db.originalRoot) + a := prefetcher.trie(common.Hash{}, db.originalRoot) + prefetcher.prefetch(common.Hash{}, db.originalRoot, [][]byte{skey.Bytes()}) + b := prefetcher.trie(common.Hash{}, db.originalRoot) cpy := prefetcher.copy() - cpy.prefetch(db.originalRoot, [][]byte{skey.Bytes()}) - cpy.prefetch(db.originalRoot, [][]byte{skey.Bytes()}) - c := cpy.trie(db.originalRoot) + cpy.prefetch(common.Hash{}, db.originalRoot, [][]byte{skey.Bytes()}) + cpy.prefetch(common.Hash{}, db.originalRoot, [][]byte{skey.Bytes()}) + c := cpy.trie(common.Hash{}, db.originalRoot) prefetcher.close() cpy2 := cpy.copy() - cpy2.prefetch(db.originalRoot, [][]byte{skey.Bytes()}) - d := cpy2.trie(db.originalRoot) + cpy2.prefetch(common.Hash{}, db.originalRoot, [][]byte{skey.Bytes()}) + d := cpy2.trie(common.Hash{}, db.originalRoot) cpy.close() cpy2.close() if a.Hash() != b.Hash() || a.Hash() != c.Hash() || a.Hash() != d.Hash() { @@ -72,10 +72,10 @@ func TestUseAfterClose(t *testing.T) { db := filledStateDB() prefetcher := newTriePrefetcher(db.db, db.originalRoot, "") skey := common.HexToHash("aaa") - prefetcher.prefetch(db.originalRoot, [][]byte{skey.Bytes()}) - a := prefetcher.trie(db.originalRoot) + prefetcher.prefetch(common.Hash{}, db.originalRoot, [][]byte{skey.Bytes()}) + a := prefetcher.trie(common.Hash{}, db.originalRoot) prefetcher.close() - b := prefetcher.trie(db.originalRoot) + b := prefetcher.trie(common.Hash{}, db.originalRoot) if a == nil { t.Fatal("Prefetching before close should not return nil") } @@ -88,13 +88,13 @@ func TestCopyClose(t *testing.T) { db := filledStateDB() prefetcher := newTriePrefetcher(db.db, db.originalRoot, "") skey := common.HexToHash("aaa") - prefetcher.prefetch(db.originalRoot, [][]byte{skey.Bytes()}) + prefetcher.prefetch(common.Hash{}, db.originalRoot, [][]byte{skey.Bytes()}) cpy := prefetcher.copy() - a := prefetcher.trie(db.originalRoot) - b := cpy.trie(db.originalRoot) + a := prefetcher.trie(common.Hash{}, db.originalRoot) + b := cpy.trie(common.Hash{}, db.originalRoot) prefetcher.close() - c := prefetcher.trie(db.originalRoot) - d := cpy.trie(db.originalRoot) + c := prefetcher.trie(common.Hash{}, db.originalRoot) + d := cpy.trie(common.Hash{}, db.originalRoot) if a == nil { t.Fatal("Prefetching before close should not return nil") } diff --git a/core/tx_pool.go b/core/tx_pool.go index 81a726bae4abc..2f21fd09fa07a 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -1603,10 +1603,6 @@ func (as *accountSet) contains(addr common.Address) bool { return exist } -func (as *accountSet) empty() bool { - return len(as.accounts) == 0 -} - // containsTx checks if the sender of a given tx is within the set. If the sender // cannot be derived, this method returns false. func (as *accountSet) containsTx(tx *types.Transaction) bool { diff --git a/core/types/block.go b/core/types/block.go index 8386c4c440e81..589a34cef6b51 100644 --- a/core/types/block.go +++ b/core/types/block.go @@ -172,10 +172,6 @@ type Block struct { hash atomic.Value size atomic.Value - // Td is used by package core to store the total difficulty - // of the chain up to and including the block. - td *big.Int - // These fields are used by package eth to track // inter-peer block relay. ReceivedAt time.Time @@ -197,7 +193,7 @@ type extblock struct { // are ignored and set to values derived from the given txs, uncles // and receipts. func NewBlock(header *Header, txs []*Transaction, uncles []*Header, receipts []*Receipt, hasher TrieHasher) *Block { - b := &Block{header: CopyHeader(header), td: new(big.Int)} + b := &Block{header: CopyHeader(header)} // TODO: panic if len(txs) != len(receipts) if len(txs) == 0 { diff --git a/core/types/hashing_test.go b/core/types/hashing_test.go index de71ee41a47d7..44726c9cbb9ff 100644 --- a/core/types/hashing_test.go +++ b/core/types/hashing_test.go @@ -39,8 +39,7 @@ func TestDeriveSha(t *testing.T) { t.Fatal(err) } for len(txs) < 1000 { - tr, _ := trie.New(common.Hash{}, trie.NewDatabase(rawdb.NewMemoryDatabase())) - exp := types.DeriveSha(txs, tr) + exp := types.DeriveSha(txs, trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase()))) got := types.DeriveSha(txs, trie.NewStackTrie(nil)) if !bytes.Equal(got[:], exp[:]) { t.Fatalf("%d txs: got %x exp %x", len(txs), got, exp) @@ -87,8 +86,7 @@ func BenchmarkDeriveSha200(b *testing.B) { b.ResetTimer() b.ReportAllocs() for i := 0; i < b.N; i++ { - tr, _ := trie.New(common.Hash{}, trie.NewDatabase(rawdb.NewMemoryDatabase())) - exp = types.DeriveSha(txs, tr) + exp = types.DeriveSha(txs, trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase()))) } }) @@ -109,8 +107,7 @@ func TestFuzzDeriveSha(t *testing.T) { rndSeed := mrand.Int() for i := 0; i < 10; i++ { seed := rndSeed + i - tr, _ := trie.New(common.Hash{}, trie.NewDatabase(rawdb.NewMemoryDatabase())) - exp := types.DeriveSha(newDummy(i), tr) + exp := types.DeriveSha(newDummy(i), trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase()))) got := types.DeriveSha(newDummy(i), trie.NewStackTrie(nil)) if !bytes.Equal(got[:], exp[:]) { printList(newDummy(seed)) @@ -138,8 +135,7 @@ func TestDerivableList(t *testing.T) { }, } for i, tc := range tcs[1:] { - tr, _ := trie.New(common.Hash{}, trie.NewDatabase(rawdb.NewMemoryDatabase())) - exp := types.DeriveSha(flatList(tc), tr) + exp := types.DeriveSha(flatList(tc), trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase()))) got := types.DeriveSha(flatList(tc), trie.NewStackTrie(nil)) if !bytes.Equal(got[:], exp[:]) { t.Fatalf("case %d: got %x exp %x", i, got, exp) diff --git a/core/types/log.go b/core/types/log.go index ee323ba868041..eb30957b12788 100644 --- a/core/types/log.go +++ b/core/types/log.go @@ -98,8 +98,8 @@ func (l *Log) DecodeRLP(s *rlp.Stream) error { return err } -// LogForStorage is a wrapper around a Log that flattens and parses the entire content of -// a log including non-consensus fields. +// LogForStorage is a wrapper around a Log that handles +// backward compatibility with prior storage formats. type LogForStorage Log // EncodeRLP implements rlp.Encoder. diff --git a/core/types/receipt.go b/core/types/receipt.go index da8537014663a..e42caf34e5a6c 100644 --- a/core/types/receipt.go +++ b/core/types/receipt.go @@ -271,8 +271,8 @@ func (r *Receipt) Size() common.StorageSize { return size } -// ReceiptForStorage is a wrapper around a Receipt that flattens and parses the -// entire content of a receipt, as opposed to only the consensus fields originally. +// ReceiptForStorage is a wrapper around a Receipt with RLP serialization +// that omits the Bloom field and deserialization that re-computes it. type ReceiptForStorage Receipt // EncodeRLP implements rlp.Encoder, and flattens all content fields of a receipt diff --git a/core/types/transaction.go b/core/types/transaction.go index d9d610c45f610..8bd7bac0b65cf 100644 --- a/core/types/transaction.go +++ b/core/types/transaction.go @@ -436,6 +436,24 @@ func TxDifference(a, b Transactions) Transactions { return keep } +// HashDifference returns a new set which is the difference between a and b. +func HashDifference(a, b []common.Hash) []common.Hash { + keep := make([]common.Hash, 0, len(a)) + + remove := make(map[common.Hash]struct{}) + for _, hash := range b { + remove[hash] = struct{}{} + } + + for _, hash := range a { + if _, ok := remove[hash]; !ok { + keep = append(keep, hash) + } + } + + return keep +} + // TxByNonce implements the sort interface to allow sorting a list of transactions // by their nonces. This is usually only useful for sorting transactions from a // single account, otherwise a nonce comparison doesn't make much sense. diff --git a/core/vm/analysis.go b/core/vm/analysis.go index 3733bab6a7c07..4aa8cfe70f11d 100644 --- a/core/vm/analysis.go +++ b/core/vm/analysis.go @@ -76,7 +76,7 @@ func codeBitmapInternal(code, bits bitvec) bitvec { for pc := uint64(0); pc < uint64(len(code)); { op := OpCode(code[pc]) pc++ - if op < PUSH1 || op > PUSH32 { + if int8(op) < int8(PUSH1) { // If not PUSH (the int8(op) > int(PUSH32) is always false). continue } numbits := op - PUSH1 + 1 diff --git a/core/vm/contracts.go b/core/vm/contracts.go index 79f1a3611680f..1b832b638695b 100644 --- a/core/vm/contracts.go +++ b/core/vm/contracts.go @@ -29,8 +29,6 @@ import ( "github.com/ethereum/go-ethereum/crypto/bls12381" "github.com/ethereum/go-ethereum/crypto/bn256" "github.com/ethereum/go-ethereum/params" - - //lint:ignore SA1019 Needed for precompile "golang.org/x/crypto/ripemd160" ) diff --git a/core/vm/instructions_test.go b/core/vm/instructions_test.go index f0fa4811ca7b5..fb0fcc1da49db 100644 --- a/core/vm/instructions_test.go +++ b/core/vm/instructions_test.go @@ -229,32 +229,32 @@ func TestAddMod(t *testing.T) { } } -// getResult is a convenience function to generate the expected values -func getResult(args []*twoOperandParams, opFn executionFunc) []TwoOperandTestcase { - var ( - env = NewEVM(BlockContext{}, TxContext{}, nil, params.TestChainConfig, Config{}) - stack = newstack() - pc = uint64(0) - interpreter = env.interpreter - ) - result := make([]TwoOperandTestcase, len(args)) - for i, param := range args { - x := new(uint256.Int).SetBytes(common.Hex2Bytes(param.x)) - y := new(uint256.Int).SetBytes(common.Hex2Bytes(param.y)) - stack.push(x) - stack.push(y) - opFn(&pc, interpreter, &ScopeContext{nil, stack, nil}) - actual := stack.pop() - result[i] = TwoOperandTestcase{param.x, param.y, fmt.Sprintf("%064x", actual)} - } - return result -} - // utility function to fill the json-file with testcases // Enable this test to generate the 'testcases_xx.json' files func TestWriteExpectedValues(t *testing.T) { t.Skip("Enable this test to create json test cases.") + // getResult is a convenience function to generate the expected values + getResult := func(args []*twoOperandParams, opFn executionFunc) []TwoOperandTestcase { + var ( + env = NewEVM(BlockContext{}, TxContext{}, nil, params.TestChainConfig, Config{}) + stack = newstack() + pc = uint64(0) + interpreter = env.interpreter + ) + result := make([]TwoOperandTestcase, len(args)) + for i, param := range args { + x := new(uint256.Int).SetBytes(common.Hex2Bytes(param.x)) + y := new(uint256.Int).SetBytes(common.Hex2Bytes(param.y)) + stack.push(x) + stack.push(y) + opFn(&pc, interpreter, &ScopeContext{nil, stack, nil}) + actual := stack.pop() + result[i] = TwoOperandTestcase{param.x, param.y, fmt.Sprintf("%064x", actual)} + } + return result + } + for name, method := range twoOpMethods { data, err := json.Marshal(getResult(commonParams, method)) if err != nil { diff --git a/core/vm/runtime/runtime_test.go b/core/vm/runtime/runtime_test.go index fcaa10f1c62cc..ca4e648436955 100644 --- a/core/vm/runtime/runtime_test.go +++ b/core/vm/runtime/runtime_test.go @@ -22,7 +22,6 @@ import ( "os" "strings" "testing" - "time" "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" @@ -326,25 +325,6 @@ func TestBlockhash(t *testing.T) { } } -type stepCounter struct { - inner *logger.JSONLogger - steps int -} - -func (s *stepCounter) CaptureStart(env *vm.EVM, from common.Address, to common.Address, create bool, input []byte, gas uint64, value *big.Int) { -} - -func (s *stepCounter) CaptureFault(pc uint64, op vm.OpCode, gas, cost uint64, scope *vm.ScopeContext, depth int, err error) { -} - -func (s *stepCounter) CaptureEnd(output []byte, gasUsed uint64, t time.Duration, err error) {} - -func (s *stepCounter) CaptureState(pc uint64, op vm.OpCode, gas, cost uint64, scope *vm.ScopeContext, rData []byte, depth int, err error) { - s.steps++ - // Enable this for more output - //s.inner.CaptureState(env, pc, op, gas, cost, memory, stack, rStack, contract, depth, err) -} - // benchmarkNonModifyingCode benchmarks code, but if the code modifies the // state, this should not be used, since it does not reset the state between runs. func benchmarkNonModifyingCode(gas uint64, code []byte, name string, tracerCode string, b *testing.B) { diff --git a/crypto/blake2b/blake2b.go b/crypto/blake2b/blake2b.go index 5da50cab6f00e..7ecaab813999f 100644 --- a/crypto/blake2b/blake2b.go +++ b/crypto/blake2b/blake2b.go @@ -302,6 +302,7 @@ func appendUint64(b []byte, x uint64) []byte { return append(b, a[:]...) } +//nolint:unused,deadcode func appendUint32(b []byte, x uint32) []byte { var a [4]byte binary.BigEndian.PutUint32(a[:], x) @@ -313,6 +314,7 @@ func consumeUint64(b []byte) ([]byte, uint64) { return b[8:], x } +//nolint:unused,deadcode func consumeUint32(b []byte) ([]byte, uint32) { x := binary.BigEndian.Uint32(b) return b[4:], x diff --git a/crypto/blake2b/blake2b_generic.go b/crypto/blake2b/blake2b_generic.go index 35c40cc924f8d..61e678fdf5763 100644 --- a/crypto/blake2b/blake2b_generic.go +++ b/crypto/blake2b/blake2b_generic.go @@ -25,6 +25,7 @@ var precomputed = [10][16]byte{ {10, 8, 7, 1, 2, 4, 6, 5, 15, 9, 3, 13, 11, 14, 12, 0}, } +// nolint:unused,deadcode func hashBlocksGeneric(h *[8]uint64, c *[2]uint64, flag uint64, blocks []byte) { var m [16]uint64 c0, c1 := c[0], c[1] diff --git a/crypto/blake2b/blake2b_test.go b/crypto/blake2b/blake2b_test.go index 9e7297da160f3..9d24444a27b7e 100644 --- a/crypto/blake2b/blake2b_test.go +++ b/crypto/blake2b/blake2b_test.go @@ -14,14 +14,6 @@ import ( "testing" ) -func fromHex(s string) []byte { - b, err := hex.DecodeString(s) - if err != nil { - panic(err) - } - return b -} - func TestHashes(t *testing.T) { defer func(sse4, avx, avx2 bool) { useSSE4, useAVX, useAVX2 = sse4, avx, avx2 diff --git a/crypto/bn256/cloudflare/gfp_decl.go b/crypto/bn256/cloudflare/gfp_decl.go index ec4018e88a0c0..cf7f5654239fa 100644 --- a/crypto/bn256/cloudflare/gfp_decl.go +++ b/crypto/bn256/cloudflare/gfp_decl.go @@ -10,7 +10,7 @@ import ( "golang.org/x/sys/cpu" ) -//nolint:varcheck +//nolint:varcheck,unused,deadcode var hasBMI2 = cpu.X86.HasBMI2 // go:noescape diff --git a/eth/api.go b/eth/api.go index ef69acb76eb45..818df89c07c94 100644 --- a/eth/api.go +++ b/eth/api.go @@ -52,17 +52,17 @@ func NewPublicEthereumAPI(e *Ethereum) *PublicEthereumAPI { return &PublicEthereumAPI{e} } -// Etherbase is the address that mining rewards will be send to +// Etherbase is the address that mining rewards will be send to. func (api *PublicEthereumAPI) Etherbase() (common.Address, error) { return api.e.Etherbase() } -// Coinbase is the address that mining rewards will be send to (alias for Etherbase) +// Coinbase is the address that mining rewards will be send to (alias for Etherbase). func (api *PublicEthereumAPI) Coinbase() (common.Address, error) { return api.Etherbase() } -// Hashrate returns the POW hashrate +// Hashrate returns the POW hashrate. func (api *PublicEthereumAPI) Hashrate() hexutil.Uint64 { return hexutil.Uint64(api.e.Miner().Hashrate()) } @@ -136,7 +136,7 @@ func (api *PrivateMinerAPI) SetGasLimit(gasLimit hexutil.Uint64) bool { return true } -// SetEtherbase sets the etherbase of the miner +// SetEtherbase sets the etherbase of the miner. func (api *PrivateMinerAPI) SetEtherbase(etherbase common.Address) bool { api.e.SetEtherbase(etherbase) return true @@ -160,7 +160,7 @@ func NewPrivateAdminAPI(eth *Ethereum) *PrivateAdminAPI { } // ExportChain exports the current blockchain into a local file, -// or a range of blocks if first and last are non-nil +// or a range of blocks if first and last are non-nil. func (api *PrivateAdminAPI) ExportChain(file string, first *uint64, last *uint64) (bool, error) { if first == nil && last != nil { return false, errors.New("last cannot be specified without first") @@ -328,7 +328,7 @@ type BadBlockArgs struct { } // GetBadBlocks returns a list of the last 'bad blocks' that the client has seen on the network -// and returns them as a JSON list of block-hashes +// and returns them as a JSON list of block hashes. func (api *PrivateDebugAPI) GetBadBlocks(ctx context.Context) ([]*BadBlockArgs, error) { var ( err error @@ -361,7 +361,7 @@ func (api *PrivateDebugAPI) GetBadBlocks(ctx context.Context) ([]*BadBlockArgs, const AccountRangeMaxResults = 256 // AccountRange enumerates all accounts in the given block and start point in paging request -func (api *PublicDebugAPI) AccountRange(blockNrOrHash rpc.BlockNumberOrHash, start []byte, maxResults int, nocode, nostorage, incompletes bool) (state.IteratorDump, error) { +func (api *PublicDebugAPI) AccountRange(blockNrOrHash rpc.BlockNumberOrHash, start hexutil.Bytes, maxResults int, nocode, nostorage, incompletes bool) (state.IteratorDump, error) { var stateDb *state.StateDB var err error @@ -529,11 +529,11 @@ func (api *PrivateDebugAPI) getModifiedAccounts(startBlock, endBlock *types.Bloc } triedb := api.eth.BlockChain().StateCache().TrieDB() - oldTrie, err := trie.NewSecure(startBlock.Root(), triedb) + oldTrie, err := trie.NewSecure(common.Hash{}, startBlock.Root(), triedb) if err != nil { return nil, err } - newTrie, err := trie.NewSecure(endBlock.Root(), triedb) + newTrie, err := trie.NewSecure(common.Hash{}, endBlock.Root(), triedb) if err != nil { return nil, err } diff --git a/eth/backend.go b/eth/backend.go index 36d4938db0e69..e36e789ee93cc 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -22,6 +22,7 @@ import ( "fmt" "math/big" "runtime" + "strings" "sync" "sync/atomic" "time" @@ -136,11 +137,17 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { if err != nil { return nil, err } - chainConfig, genesisHash, genesisErr := core.SetupGenesisBlockWithOverride(chainDb, config.Genesis, config.OverrideArrowGlacier, config.OverrideTerminalTotalDifficulty) + chainConfig, genesisHash, genesisErr := core.SetupGenesisBlockWithOverride(chainDb, config.Genesis, config.OverrideGrayGlacier, config.OverrideTerminalTotalDifficulty) if _, ok := genesisErr.(*params.ConfigCompatError); genesisErr != nil && !ok { return nil, genesisErr } - log.Info("Initialised chain configuration", "config", chainConfig) + log.Info("") + log.Info(strings.Repeat("-", 153)) + for _, line := range strings.Split(chainConfig.String(), "\n") { + log.Info(line) + } + log.Info(strings.Repeat("-", 153)) + log.Info("") if err := pruner.RecoverPruning(stack.ResolvePath(""), chainDb, stack.ResolvePath(config.TrieCleanCacheJournal)); err != nil { log.Error("Failed to recover state", "error", err) diff --git a/eth/catalyst/api.go b/eth/catalyst/api.go index 108ec412d9e3f..166931e381346 100644 --- a/eth/catalyst/api.go +++ b/eth/catalyst/api.go @@ -47,13 +47,6 @@ func Register(stack *node.Node, backend *eth.Ethereum) error { Public: true, Authenticated: true, }, - { - Namespace: "engine", - Version: "1.0", - Service: NewConsensusAPI(backend), - Public: true, - Authenticated: false, - }, }) return nil } @@ -138,10 +131,14 @@ func (api *ConsensusAPI) ForkchoiceUpdatedV1(update beacon.ForkchoiceStateV1, pa log.Error("TDs unavailable for TTD check", "number", block.NumberU64(), "hash", update.HeadBlockHash, "td", td, "parent", block.ParentHash(), "ptd", ptd) return beacon.STATUS_INVALID, errors.New("TDs unavailable for TDD check") } - if td.Cmp(ttd) < 0 || (block.NumberU64() > 0 && ptd.Cmp(ttd) > 0) { + if td.Cmp(ttd) < 0 { log.Error("Refusing beacon update to pre-merge", "number", block.NumberU64(), "hash", update.HeadBlockHash, "diff", block.Difficulty(), "age", common.PrettyAge(time.Unix(int64(block.Time()), 0))) return beacon.ForkChoiceResponse{PayloadStatus: beacon.INVALID_TERMINAL_BLOCK, PayloadID: nil}, nil } + if block.NumberU64() > 0 && ptd.Cmp(ttd) >= 0 { + log.Error("Parent block is already post-ttd", "number", block.NumberU64(), "hash", update.HeadBlockHash, "diff", block.Difficulty(), "age", common.PrettyAge(time.Unix(int64(block.Time()), 0))) + return beacon.ForkChoiceResponse{PayloadStatus: beacon.INVALID_TERMINAL_BLOCK, PayloadID: nil}, nil + } } if rawdb.ReadCanonicalHash(api.eth.ChainDb(), block.NumberU64()) != update.HeadBlockHash { @@ -295,11 +292,16 @@ func (api *ConsensusAPI) NewPayloadV1(params beacon.ExecutableDataV1) (beacon.Pa // We have an existing parent, do some sanity checks to avoid the beacon client // triggering too early var ( - td = api.eth.BlockChain().GetTd(parent.Hash(), parent.NumberU64()) - ttd = api.eth.BlockChain().Config().TerminalTotalDifficulty + ptd = api.eth.BlockChain().GetTd(parent.Hash(), parent.NumberU64()) + ttd = api.eth.BlockChain().Config().TerminalTotalDifficulty + gptd = api.eth.BlockChain().GetTd(parent.ParentHash(), parent.NumberU64()-1) ) - if td.Cmp(ttd) < 0 { - log.Warn("Ignoring pre-merge payload", "number", params.Number, "hash", params.BlockHash, "td", td, "ttd", ttd) + if ptd.Cmp(ttd) < 0 { + log.Warn("Ignoring pre-merge payload", "number", params.Number, "hash", params.BlockHash, "td", ptd, "ttd", ttd) + return beacon.INVALID_TERMINAL_BLOCK, nil + } + if parent.Difficulty().BitLen() > 0 && gptd != nil && gptd.Cmp(ttd) >= 0 { + log.Error("Ignoring pre-merge parent block", "number", params.Number, "hash", params.BlockHash, "td", ptd, "ttd", ttd) return beacon.INVALID_TERMINAL_BLOCK, nil } if block.Time() <= parent.Time() { @@ -345,7 +347,12 @@ func computePayloadId(headBlockHash common.Hash, params *beacon.PayloadAttribute func (api *ConsensusAPI) invalid(err error, latestValid *types.Block) beacon.PayloadStatusV1 { currentHash := api.eth.BlockChain().CurrentBlock().Hash() if latestValid != nil { - currentHash = latestValid.Hash() + // Set latest valid hash to 0x0 if parent is PoW block + currentHash = common.Hash{} + if latestValid.Difficulty().BitLen() == 0 { + // Otherwise set latest valid hash to parent hash + currentHash = latestValid.Hash() + } } errorMsg := err.Error() return beacon.PayloadStatusV1{Status: beacon.INVALID, LatestValidHash: ¤tHash, ValidationError: &errorMsg} diff --git a/eth/catalyst/api_test.go b/eth/catalyst/api_test.go index 415506d58e507..6171c14c432c8 100644 --- a/eth/catalyst/api_test.go +++ b/eth/catalyst/api_test.go @@ -205,7 +205,6 @@ func checkLogEvents(t *testing.T, logsCh <-chan []*types.Log, rmLogsCh <-chan co func TestInvalidPayloadTimestamp(t *testing.T) { genesis, preMergeBlocks := generatePreMergeChain(10) n, ethservice := startEthService(t, genesis, preMergeBlocks) - ethservice.Merger().ReachTTD() defer n.Close() var ( api = NewConsensusAPI(ethservice) @@ -250,7 +249,6 @@ func TestInvalidPayloadTimestamp(t *testing.T) { func TestEth2NewBlock(t *testing.T) { genesis, preMergeBlocks := generatePreMergeChain(10) n, ethservice := startEthService(t, genesis, preMergeBlocks) - ethservice.Merger().ReachTTD() defer n.Close() var ( @@ -427,7 +425,6 @@ func startEthService(t *testing.T, genesis *core.Genesis, blocks []*types.Block) func TestFullAPI(t *testing.T) { genesis, preMergeBlocks := generatePreMergeChain(10) n, ethservice := startEthService(t, genesis, preMergeBlocks) - ethservice.Merger().ReachTTD() defer n.Close() var ( parent = ethservice.BlockChain().CurrentBlock() @@ -480,7 +477,6 @@ func setupBlocks(t *testing.T, ethservice *eth.Ethereum, n int, parent *types.Bl func TestExchangeTransitionConfig(t *testing.T) { genesis, preMergeBlocks := generatePreMergeChain(10) n, ethservice := startEthService(t, genesis, preMergeBlocks) - ethservice.Merger().ReachTTD() defer n.Close() var ( api = NewConsensusAPI(ethservice) @@ -543,7 +539,6 @@ CommonAncestor◄─▲── P1 ◄── P2 ◄─ P3 ◄─ ... ◄─ Pn func TestNewPayloadOnInvalidChain(t *testing.T) { genesis, preMergeBlocks := generatePreMergeChain(10) n, ethservice := startEthService(t, genesis, preMergeBlocks) - ethservice.Merger().ReachTTD() defer n.Close() var ( @@ -618,7 +613,6 @@ func assembleBlock(api *ConsensusAPI, parentHash common.Hash, params *beacon.Pay func TestEmptyBlocks(t *testing.T) { genesis, preMergeBlocks := generatePreMergeChain(10) n, ethservice := startEthService(t, genesis, preMergeBlocks) - ethservice.Merger().ReachTTD() defer n.Close() commonAncestor := ethservice.BlockChain().CurrentBlock() @@ -653,7 +647,8 @@ func TestEmptyBlocks(t *testing.T) { if status.Status != beacon.INVALID { t.Errorf("invalid status: expected INVALID got: %v", status.Status) } - expected := commonAncestor.Hash() + // Expect 0x0 on INVALID block on top of PoW block + expected := common.Hash{} if !bytes.Equal(status.LatestValidHash[:], expected[:]) { t.Fatalf("invalid LVH: got %v want %v", status.LatestValidHash, expected) } @@ -734,8 +729,6 @@ func TestTrickRemoteBlockCache(t *testing.T) { genesis, preMergeBlocks := generatePreMergeChain(10) nodeA, ethserviceA := startEthService(t, genesis, preMergeBlocks) nodeB, ethserviceB := startEthService(t, genesis, preMergeBlocks) - ethserviceA.Merger().ReachTTD() - ethserviceB.Merger().ReachTTD() defer nodeA.Close() defer nodeB.Close() for nodeB.Server().NodeInfo().Ports.Listener == 0 { @@ -794,3 +787,75 @@ func TestTrickRemoteBlockCache(t *testing.T) { time.Sleep(100 * time.Millisecond) } } + +func TestInvalidBloom(t *testing.T) { + genesis, preMergeBlocks := generatePreMergeChain(10) + n, ethservice := startEthService(t, genesis, preMergeBlocks) + ethservice.Merger().ReachTTD() + defer n.Close() + + commonAncestor := ethservice.BlockChain().CurrentBlock() + api := NewConsensusAPI(ethservice) + + // Setup 10 blocks on the canonical chain + setupBlocks(t, ethservice, 10, commonAncestor, func(parent *types.Block) {}) + + // (1) check LatestValidHash by sending a normal payload (P1'') + payload := getNewPayload(t, api, commonAncestor) + payload.LogsBloom = append(payload.LogsBloom, byte(1)) + status, err := api.NewPayloadV1(*payload) + if err != nil { + t.Fatal(err) + } + if status.Status != beacon.INVALIDBLOCKHASH { + t.Errorf("invalid status: expected VALID got: %v", status.Status) + } +} + +func TestNewPayloadOnInvalidTerminalBlock(t *testing.T) { + genesis, preMergeBlocks := generatePreMergeChain(100) + fmt.Println(genesis.Config.TerminalTotalDifficulty) + genesis.Config.TerminalTotalDifficulty = preMergeBlocks[0].Difficulty() //.Sub(genesis.Config.TerminalTotalDifficulty, preMergeBlocks[len(preMergeBlocks)-1].Difficulty()) + + fmt.Println(genesis.Config.TerminalTotalDifficulty) + n, ethservice := startEthService(t, genesis, preMergeBlocks) + defer n.Close() + + var ( + api = NewConsensusAPI(ethservice) + parent = preMergeBlocks[len(preMergeBlocks)-1] + ) + + // Test parent already post TTD in FCU + fcState := beacon.ForkchoiceStateV1{ + HeadBlockHash: parent.Hash(), + SafeBlockHash: common.Hash{}, + FinalizedBlockHash: common.Hash{}, + } + resp, err := api.ForkchoiceUpdatedV1(fcState, nil) + if err != nil { + t.Fatalf("error sending forkchoice, err=%v", err) + } + if resp.PayloadStatus != beacon.INVALID_TERMINAL_BLOCK { + t.Fatalf("error sending invalid forkchoice, invalid status: %v", resp.PayloadStatus.Status) + } + + // Test parent already post TTD in NewPayload + params := beacon.PayloadAttributesV1{ + Timestamp: parent.Time() + 1, + Random: crypto.Keccak256Hash([]byte{byte(1)}), + SuggestedFeeRecipient: parent.Coinbase(), + } + empty, err := api.eth.Miner().GetSealingBlockSync(parent.Hash(), params.Timestamp, params.SuggestedFeeRecipient, params.Random, true) + if err != nil { + t.Fatalf("error preparing payload, err=%v", err) + } + data := *beacon.BlockToExecutableData(empty) + resp2, err := api.NewPayloadV1(data) + if err != nil { + t.Fatalf("error sending NewPayload, err=%v", err) + } + if resp2 != beacon.INVALID_TERMINAL_BLOCK { + t.Fatalf("error sending invalid forkchoice, invalid status: %v", resp.PayloadStatus.Status) + } +} diff --git a/eth/discovery.go b/eth/discovery.go deleted file mode 100644 index f7c85b4c5d3ba..0000000000000 --- a/eth/discovery.go +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright 2020 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package eth - -import ( - "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/forkid" - "github.com/ethereum/go-ethereum/p2p/enode" - "github.com/ethereum/go-ethereum/rlp" -) - -// ethEntry is the "eth" ENR entry which advertises eth protocol -// on the discovery network. -type ethEntry struct { - ForkID forkid.ID // Fork identifier per EIP-2124 - - // Ignore additional fields (for forward compatibility). - Rest []rlp.RawValue `rlp:"tail"` -} - -// ENRKey implements enr.Entry. -func (e ethEntry) ENRKey() string { - return "eth" -} - -// startEthEntryUpdate starts the ENR updater loop. -func (eth *Ethereum) startEthEntryUpdate(ln *enode.LocalNode) { - var newHead = make(chan core.ChainHeadEvent, 10) - sub := eth.blockchain.SubscribeChainHeadEvent(newHead) - - go func() { - defer sub.Unsubscribe() - for { - select { - case <-newHead: - ln.Set(eth.currentEthEntry()) - case <-sub.Err(): - // Would be nice to sync with eth.Stop, but there is no - // good way to do that. - return - } - } - }() -} - -func (eth *Ethereum) currentEthEntry() *ethEntry { - return ðEntry{ForkID: forkid.NewID(eth.blockchain.Config(), eth.blockchain.Genesis().Hash(), - eth.blockchain.CurrentHeader().Number.Uint64())} -} diff --git a/eth/downloader/api.go b/eth/downloader/api.go index 2024d23deade6..645912286d574 100644 --- a/eth/downloader/api.go +++ b/eth/downloader/api.go @@ -159,7 +159,7 @@ func (s *SyncStatusSubscription) Unsubscribe() { } // SubscribeSyncStatus creates a subscription that will broadcast new synchronisation updates. -// The given channel must receive interface values, the result can either +// The given channel must receive interface values, the result can either. func (api *PublicDownloaderAPI) SubscribeSyncStatus(status chan interface{}) *SyncStatusSubscription { api.installSyncSubscription <- status return &SyncStatusSubscription{api: api, c: status} diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index c836fdd4b8cf1..1dcacba3a2670 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -131,7 +131,6 @@ type Downloader struct { pivotHeader *types.Header // Pivot block header to dynamically push the syncing state root pivotLock sync.RWMutex // Lock protecting pivot header reads from updates - snapSync bool // Whether to run state sync over the snap protocol SnapSyncer *snap.Syncer // TODO(karalabe): make private! hack for now stateSyncStart chan *stateSync diff --git a/eth/downloader/downloader_test.go b/eth/downloader/downloader_test.go index a5db037a456ca..5e77d3272d636 100644 --- a/eth/downloader/downloader_test.go +++ b/eth/downloader/downloader_test.go @@ -437,6 +437,9 @@ func assertOwnChain(t *testing.T, tester *downloadTester, length int) { func TestCanonicalSynchronisation66Full(t *testing.T) { testCanonSync(t, eth.ETH66, FullSync) } func TestCanonicalSynchronisation66Snap(t *testing.T) { testCanonSync(t, eth.ETH66, SnapSync) } func TestCanonicalSynchronisation66Light(t *testing.T) { testCanonSync(t, eth.ETH66, LightSync) } +func TestCanonicalSynchronisation67Full(t *testing.T) { testCanonSync(t, eth.ETH67, FullSync) } +func TestCanonicalSynchronisation67Snap(t *testing.T) { testCanonSync(t, eth.ETH67, SnapSync) } +func TestCanonicalSynchronisation67Light(t *testing.T) { testCanonSync(t, eth.ETH67, LightSync) } func testCanonSync(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -457,6 +460,8 @@ func testCanonSync(t *testing.T, protocol uint, mode SyncMode) { // until the cached blocks are retrieved. func TestThrottling66Full(t *testing.T) { testThrottling(t, eth.ETH66, FullSync) } func TestThrottling66Snap(t *testing.T) { testThrottling(t, eth.ETH66, SnapSync) } +func TestThrottling67Full(t *testing.T) { testThrottling(t, eth.ETH67, FullSync) } +func TestThrottling67Snap(t *testing.T) { testThrottling(t, eth.ETH67, SnapSync) } func testThrottling(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -537,6 +542,9 @@ func testThrottling(t *testing.T, protocol uint, mode SyncMode) { func TestForkedSync66Full(t *testing.T) { testForkedSync(t, eth.ETH66, FullSync) } func TestForkedSync66Snap(t *testing.T) { testForkedSync(t, eth.ETH66, SnapSync) } func TestForkedSync66Light(t *testing.T) { testForkedSync(t, eth.ETH66, LightSync) } +func TestForkedSync67Full(t *testing.T) { testForkedSync(t, eth.ETH67, FullSync) } +func TestForkedSync67Snap(t *testing.T) { testForkedSync(t, eth.ETH67, SnapSync) } +func TestForkedSync67Light(t *testing.T) { testForkedSync(t, eth.ETH67, LightSync) } func testForkedSync(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -564,6 +572,9 @@ func testForkedSync(t *testing.T, protocol uint, mode SyncMode) { func TestHeavyForkedSync66Full(t *testing.T) { testHeavyForkedSync(t, eth.ETH66, FullSync) } func TestHeavyForkedSync66Snap(t *testing.T) { testHeavyForkedSync(t, eth.ETH66, SnapSync) } func TestHeavyForkedSync66Light(t *testing.T) { testHeavyForkedSync(t, eth.ETH66, LightSync) } +func TestHeavyForkedSync67Full(t *testing.T) { testHeavyForkedSync(t, eth.ETH67, FullSync) } +func TestHeavyForkedSync67Snap(t *testing.T) { testHeavyForkedSync(t, eth.ETH67, SnapSync) } +func TestHeavyForkedSync67Light(t *testing.T) { testHeavyForkedSync(t, eth.ETH67, LightSync) } func testHeavyForkedSync(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -593,6 +604,9 @@ func testHeavyForkedSync(t *testing.T, protocol uint, mode SyncMode) { func TestBoundedForkedSync66Full(t *testing.T) { testBoundedForkedSync(t, eth.ETH66, FullSync) } func TestBoundedForkedSync66Snap(t *testing.T) { testBoundedForkedSync(t, eth.ETH66, SnapSync) } func TestBoundedForkedSync66Light(t *testing.T) { testBoundedForkedSync(t, eth.ETH66, LightSync) } +func TestBoundedForkedSync67Full(t *testing.T) { testBoundedForkedSync(t, eth.ETH67, FullSync) } +func TestBoundedForkedSync67Snap(t *testing.T) { testBoundedForkedSync(t, eth.ETH67, SnapSync) } +func TestBoundedForkedSync67Light(t *testing.T) { testBoundedForkedSync(t, eth.ETH67, LightSync) } func testBoundedForkedSync(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -627,6 +641,15 @@ func TestBoundedHeavyForkedSync66Snap(t *testing.T) { func TestBoundedHeavyForkedSync66Light(t *testing.T) { testBoundedHeavyForkedSync(t, eth.ETH66, LightSync) } +func TestBoundedHeavyForkedSync67Full(t *testing.T) { + testBoundedHeavyForkedSync(t, eth.ETH67, FullSync) +} +func TestBoundedHeavyForkedSync67Snap(t *testing.T) { + testBoundedHeavyForkedSync(t, eth.ETH67, SnapSync) +} +func TestBoundedHeavyForkedSync67Light(t *testing.T) { + testBoundedHeavyForkedSync(t, eth.ETH67, LightSync) +} func testBoundedHeavyForkedSync(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -654,6 +677,9 @@ func testBoundedHeavyForkedSync(t *testing.T, protocol uint, mode SyncMode) { func TestCancel66Full(t *testing.T) { testCancel(t, eth.ETH66, FullSync) } func TestCancel66Snap(t *testing.T) { testCancel(t, eth.ETH66, SnapSync) } func TestCancel66Light(t *testing.T) { testCancel(t, eth.ETH66, LightSync) } +func TestCancel67Full(t *testing.T) { testCancel(t, eth.ETH67, FullSync) } +func TestCancel67Snap(t *testing.T) { testCancel(t, eth.ETH67, SnapSync) } +func TestCancel67Light(t *testing.T) { testCancel(t, eth.ETH67, LightSync) } func testCancel(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -681,6 +707,9 @@ func testCancel(t *testing.T, protocol uint, mode SyncMode) { func TestMultiSynchronisation66Full(t *testing.T) { testMultiSynchronisation(t, eth.ETH66, FullSync) } func TestMultiSynchronisation66Snap(t *testing.T) { testMultiSynchronisation(t, eth.ETH66, SnapSync) } func TestMultiSynchronisation66Light(t *testing.T) { testMultiSynchronisation(t, eth.ETH66, LightSync) } +func TestMultiSynchronisation67Full(t *testing.T) { testMultiSynchronisation(t, eth.ETH67, FullSync) } +func TestMultiSynchronisation67Snap(t *testing.T) { testMultiSynchronisation(t, eth.ETH67, SnapSync) } +func TestMultiSynchronisation67Light(t *testing.T) { testMultiSynchronisation(t, eth.ETH67, LightSync) } func testMultiSynchronisation(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -705,6 +734,9 @@ func testMultiSynchronisation(t *testing.T, protocol uint, mode SyncMode) { func TestMultiProtoSynchronisation66Full(t *testing.T) { testMultiProtoSync(t, eth.ETH66, FullSync) } func TestMultiProtoSynchronisation66Snap(t *testing.T) { testMultiProtoSync(t, eth.ETH66, SnapSync) } func TestMultiProtoSynchronisation66Light(t *testing.T) { testMultiProtoSync(t, eth.ETH66, LightSync) } +func TestMultiProtoSynchronisation67Full(t *testing.T) { testMultiProtoSync(t, eth.ETH67, FullSync) } +func TestMultiProtoSynchronisation67Snap(t *testing.T) { testMultiProtoSync(t, eth.ETH67, SnapSync) } +func TestMultiProtoSynchronisation67Light(t *testing.T) { testMultiProtoSync(t, eth.ETH67, LightSync) } func testMultiProtoSync(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -715,7 +747,7 @@ func testMultiProtoSync(t *testing.T, protocol uint, mode SyncMode) { // Create peers of every type tester.newPeer("peer 66", eth.ETH66, chain.blocks[1:]) - //tester.newPeer("peer 65", eth.ETH67, chain.blocks[1:) + tester.newPeer("peer 67", eth.ETH67, chain.blocks[1:]) // Synchronise with the requested peer and make sure all blocks were retrieved if err := tester.sync(fmt.Sprintf("peer %d", protocol), nil, mode); err != nil { @@ -724,7 +756,7 @@ func testMultiProtoSync(t *testing.T, protocol uint, mode SyncMode) { assertOwnChain(t, tester, len(chain.blocks)) // Check that no peers have been dropped off - for _, version := range []int{66} { + for _, version := range []int{66, 67} { peer := fmt.Sprintf("peer %d", version) if _, ok := tester.peers[peer]; !ok { t.Errorf("%s dropped", peer) @@ -737,6 +769,9 @@ func testMultiProtoSync(t *testing.T, protocol uint, mode SyncMode) { func TestEmptyShortCircuit66Full(t *testing.T) { testEmptyShortCircuit(t, eth.ETH66, FullSync) } func TestEmptyShortCircuit66Snap(t *testing.T) { testEmptyShortCircuit(t, eth.ETH66, SnapSync) } func TestEmptyShortCircuit66Light(t *testing.T) { testEmptyShortCircuit(t, eth.ETH66, LightSync) } +func TestEmptyShortCircuit67Full(t *testing.T) { testEmptyShortCircuit(t, eth.ETH67, FullSync) } +func TestEmptyShortCircuit67Snap(t *testing.T) { testEmptyShortCircuit(t, eth.ETH67, SnapSync) } +func TestEmptyShortCircuit67Light(t *testing.T) { testEmptyShortCircuit(t, eth.ETH67, LightSync) } func testEmptyShortCircuit(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -785,6 +820,9 @@ func testEmptyShortCircuit(t *testing.T, protocol uint, mode SyncMode) { func TestMissingHeaderAttack66Full(t *testing.T) { testMissingHeaderAttack(t, eth.ETH66, FullSync) } func TestMissingHeaderAttack66Snap(t *testing.T) { testMissingHeaderAttack(t, eth.ETH66, SnapSync) } func TestMissingHeaderAttack66Light(t *testing.T) { testMissingHeaderAttack(t, eth.ETH66, LightSync) } +func TestMissingHeaderAttack67Full(t *testing.T) { testMissingHeaderAttack(t, eth.ETH67, FullSync) } +func TestMissingHeaderAttack67Snap(t *testing.T) { testMissingHeaderAttack(t, eth.ETH67, SnapSync) } +func TestMissingHeaderAttack67Light(t *testing.T) { testMissingHeaderAttack(t, eth.ETH67, LightSync) } func testMissingHeaderAttack(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -811,6 +849,9 @@ func testMissingHeaderAttack(t *testing.T, protocol uint, mode SyncMode) { func TestShiftedHeaderAttack66Full(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH66, FullSync) } func TestShiftedHeaderAttack66Snap(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH66, SnapSync) } func TestShiftedHeaderAttack66Light(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH66, LightSync) } +func TestShiftedHeaderAttack67Full(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH67, FullSync) } +func TestShiftedHeaderAttack67Snap(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH67, SnapSync) } +func TestShiftedHeaderAttack67Light(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH67, LightSync) } func testShiftedHeaderAttack(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -837,6 +878,7 @@ func testShiftedHeaderAttack(t *testing.T, protocol uint, mode SyncMode) { // for various failure scenarios. Afterwards a full sync is attempted to make // sure no state was corrupted. func TestInvalidHeaderRollback66Snap(t *testing.T) { testInvalidHeaderRollback(t, eth.ETH66, SnapSync) } +func TestInvalidHeaderRollback67Snap(t *testing.T) { testInvalidHeaderRollback(t, eth.ETH67, SnapSync) } func testInvalidHeaderRollback(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -923,6 +965,15 @@ func TestHighTDStarvationAttack66Snap(t *testing.T) { func TestHighTDStarvationAttack66Light(t *testing.T) { testHighTDStarvationAttack(t, eth.ETH66, LightSync) } +func TestHighTDStarvationAttack67Full(t *testing.T) { + testHighTDStarvationAttack(t, eth.ETH67, FullSync) +} +func TestHighTDStarvationAttack67Snap(t *testing.T) { + testHighTDStarvationAttack(t, eth.ETH67, SnapSync) +} +func TestHighTDStarvationAttack67Light(t *testing.T) { + testHighTDStarvationAttack(t, eth.ETH67, LightSync) +} func testHighTDStarvationAttack(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -937,6 +988,7 @@ func testHighTDStarvationAttack(t *testing.T, protocol uint, mode SyncMode) { // Tests that misbehaving peers are disconnected, whilst behaving ones are not. func TestBlockHeaderAttackerDropping66(t *testing.T) { testBlockHeaderAttackerDropping(t, eth.ETH66) } +func TestBlockHeaderAttackerDropping67(t *testing.T) { testBlockHeaderAttackerDropping(t, eth.ETH67) } func testBlockHeaderAttackerDropping(t *testing.T, protocol uint) { // Define the disconnection requirement for individual hash fetch errors @@ -987,6 +1039,9 @@ func testBlockHeaderAttackerDropping(t *testing.T, protocol uint) { func TestSyncProgress66Full(t *testing.T) { testSyncProgress(t, eth.ETH66, FullSync) } func TestSyncProgress66Snap(t *testing.T) { testSyncProgress(t, eth.ETH66, SnapSync) } func TestSyncProgress66Light(t *testing.T) { testSyncProgress(t, eth.ETH66, LightSync) } +func TestSyncProgress67Full(t *testing.T) { testSyncProgress(t, eth.ETH67, FullSync) } +func TestSyncProgress67Snap(t *testing.T) { testSyncProgress(t, eth.ETH67, SnapSync) } +func TestSyncProgress67Light(t *testing.T) { testSyncProgress(t, eth.ETH67, LightSync) } func testSyncProgress(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -1064,6 +1119,9 @@ func checkProgress(t *testing.T, d *Downloader, stage string, want ethereum.Sync func TestForkedSyncProgress66Full(t *testing.T) { testForkedSyncProgress(t, eth.ETH66, FullSync) } func TestForkedSyncProgress66Snap(t *testing.T) { testForkedSyncProgress(t, eth.ETH66, SnapSync) } func TestForkedSyncProgress66Light(t *testing.T) { testForkedSyncProgress(t, eth.ETH66, LightSync) } +func TestForkedSyncProgress67Full(t *testing.T) { testForkedSyncProgress(t, eth.ETH67, FullSync) } +func TestForkedSyncProgress67Snap(t *testing.T) { testForkedSyncProgress(t, eth.ETH67, SnapSync) } +func TestForkedSyncProgress67Light(t *testing.T) { testForkedSyncProgress(t, eth.ETH67, LightSync) } func testForkedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -1135,6 +1193,9 @@ func testForkedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { func TestFailedSyncProgress66Full(t *testing.T) { testFailedSyncProgress(t, eth.ETH66, FullSync) } func TestFailedSyncProgress66Snap(t *testing.T) { testFailedSyncProgress(t, eth.ETH66, SnapSync) } func TestFailedSyncProgress66Light(t *testing.T) { testFailedSyncProgress(t, eth.ETH66, LightSync) } +func TestFailedSyncProgress67Full(t *testing.T) { testFailedSyncProgress(t, eth.ETH67, FullSync) } +func TestFailedSyncProgress67Snap(t *testing.T) { testFailedSyncProgress(t, eth.ETH67, SnapSync) } +func TestFailedSyncProgress67Light(t *testing.T) { testFailedSyncProgress(t, eth.ETH67, LightSync) } func testFailedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -1201,6 +1262,9 @@ func testFailedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { func TestFakedSyncProgress66Full(t *testing.T) { testFakedSyncProgress(t, eth.ETH66, FullSync) } func TestFakedSyncProgress66Snap(t *testing.T) { testFakedSyncProgress(t, eth.ETH66, SnapSync) } func TestFakedSyncProgress66Light(t *testing.T) { testFakedSyncProgress(t, eth.ETH66, LightSync) } +func TestFakedSyncProgress67Full(t *testing.T) { testFakedSyncProgress(t, eth.ETH67, FullSync) } +func TestFakedSyncProgress67Snap(t *testing.T) { testFakedSyncProgress(t, eth.ETH67, SnapSync) } +func TestFakedSyncProgress67Light(t *testing.T) { testFakedSyncProgress(t, eth.ETH67, LightSync) } func testFakedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -1347,6 +1411,11 @@ func TestCheckpointEnforcement66Snap(t *testing.T) { testCheckpointEnforcement(t func TestCheckpointEnforcement66Light(t *testing.T) { testCheckpointEnforcement(t, eth.ETH66, LightSync) } +func TestCheckpointEnforcement67Full(t *testing.T) { testCheckpointEnforcement(t, eth.ETH67, FullSync) } +func TestCheckpointEnforcement67Snap(t *testing.T) { testCheckpointEnforcement(t, eth.ETH67, SnapSync) } +func TestCheckpointEnforcement67Light(t *testing.T) { + testCheckpointEnforcement(t, eth.ETH67, LightSync) +} func testCheckpointEnforcement(t *testing.T, protocol uint, mode SyncMode) { // Create a new tester with a particular hard coded checkpoint block diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index 664e05aebca56..b66452070b303 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -205,8 +205,8 @@ type Config struct { // CheckpointOracle is the configuration for checkpoint oracle. CheckpointOracle *params.CheckpointOracleConfig `toml:",omitempty"` - // Arrow Glacier block override (TODO: remove after the fork) - OverrideArrowGlacier *big.Int `toml:",omitempty"` + // Gray Glacier block override (TODO: remove after the fork) + OverrideGrayGlacier *big.Int `toml:",omitempty"` // OverrideTerminalTotalDifficulty (TODO: remove after the fork) OverrideTerminalTotalDifficulty *big.Int `toml:",omitempty"` diff --git a/eth/ethconfig/gen_config.go b/eth/ethconfig/gen_config.go index 0f43c36ad4472..e714dd97ab196 100644 --- a/eth/ethconfig/gen_config.go +++ b/eth/ethconfig/gen_config.go @@ -59,7 +59,7 @@ func (c Config) MarshalTOML() (interface{}, error) { RPCTxFeeCap float64 Checkpoint *params.TrustedCheckpoint `toml:",omitempty"` CheckpointOracle *params.CheckpointOracleConfig `toml:",omitempty"` - OverrideArrowGlacier *big.Int `toml:",omitempty"` + OverrideGrayGlacier *big.Int `toml:",omitempty"` OverrideTerminalTotalDifficulty *big.Int `toml:",omitempty"` } var enc Config @@ -104,7 +104,7 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.RPCTxFeeCap = c.RPCTxFeeCap enc.Checkpoint = c.Checkpoint enc.CheckpointOracle = c.CheckpointOracle - enc.OverrideArrowGlacier = c.OverrideArrowGlacier + enc.OverrideGrayGlacier = c.OverrideGrayGlacier enc.OverrideTerminalTotalDifficulty = c.OverrideTerminalTotalDifficulty return &enc, nil } @@ -153,7 +153,7 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { RPCTxFeeCap *float64 Checkpoint *params.TrustedCheckpoint `toml:",omitempty"` CheckpointOracle *params.CheckpointOracleConfig `toml:",omitempty"` - OverrideArrowGlacier *big.Int `toml:",omitempty"` + OverrideGrayGlacier *big.Int `toml:",omitempty"` OverrideTerminalTotalDifficulty *big.Int `toml:",omitempty"` } var dec Config @@ -283,8 +283,8 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.CheckpointOracle != nil { c.CheckpointOracle = dec.CheckpointOracle } - if dec.OverrideArrowGlacier != nil { - c.OverrideArrowGlacier = dec.OverrideArrowGlacier + if dec.OverrideGrayGlacier != nil { + c.OverrideGrayGlacier = dec.OverrideGrayGlacier } if dec.OverrideTerminalTotalDifficulty != nil { c.OverrideTerminalTotalDifficulty = dec.OverrideTerminalTotalDifficulty diff --git a/eth/filters/api.go b/eth/filters/api.go index 7196e90f9eb71..ce2b0cb0b87db 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -101,8 +101,6 @@ func (api *PublicFilterAPI) timeoutLoop(timeout time.Duration) { // // It is part of the filter package because this filter can be used through the // `eth_getFilterChanges` polling method that is also used for log filters. -// -// https://eth.wiki/json-rpc/API#eth_newpendingtransactionfilter func (api *PublicFilterAPI) NewPendingTransactionFilter() rpc.ID { var ( pendingTxs = make(chan []common.Hash) @@ -171,8 +169,6 @@ func (api *PublicFilterAPI) NewPendingTransactions(ctx context.Context) (*rpc.Su // NewBlockFilter creates a filter that fetches blocks that are imported into the chain. // It is part of the filter package since polling goes with eth_getFilterChanges. -// -// https://eth.wiki/json-rpc/API#eth_newblockfilter func (api *PublicFilterAPI) NewBlockFilter() rpc.ID { var ( headers = make(chan *types.Header) @@ -257,6 +253,7 @@ func (api *PublicFilterAPI) Logs(ctx context.Context, crit FilterCriteria) (*rpc select { case logs := <-matchedLogs: for _, log := range logs { + log := log notifier.Notify(rpcSub.ID, &log) } case <-rpcSub.Err(): // client send an unsubscribe request @@ -287,8 +284,6 @@ type FilterCriteria ethereum.FilterQuery // again but with the removed property set to true. // // In case "fromBlock" > "toBlock" an error is returned. -// -// https://eth.wiki/json-rpc/API#eth_newfilter func (api *PublicFilterAPI) NewFilter(crit FilterCriteria) (rpc.ID, error) { logs := make(chan []*types.Log) logsSub, err := api.events.SubscribeLogs(ethereum.FilterQuery(crit), logs) @@ -322,8 +317,6 @@ func (api *PublicFilterAPI) NewFilter(crit FilterCriteria) (rpc.ID, error) { } // GetLogs returns logs matching the given argument that are stored within the state. -// -// https://eth.wiki/json-rpc/API#eth_getlogs func (api *PublicFilterAPI) GetLogs(ctx context.Context, crit FilterCriteria) ([]*types.Log, error) { var filter *Filter if crit.BlockHash != nil { @@ -351,8 +344,6 @@ func (api *PublicFilterAPI) GetLogs(ctx context.Context, crit FilterCriteria) ([ } // UninstallFilter removes the filter with the given filter id. -// -// https://eth.wiki/json-rpc/API#eth_uninstallfilter func (api *PublicFilterAPI) UninstallFilter(id rpc.ID) bool { api.filtersMu.Lock() f, found := api.filters[id] @@ -369,8 +360,6 @@ func (api *PublicFilterAPI) UninstallFilter(id rpc.ID) bool { // GetFilterLogs returns the logs for the filter with the given id. // If the filter could not be found an empty array of logs is returned. -// -// https://eth.wiki/json-rpc/API#eth_getfilterlogs func (api *PublicFilterAPI) GetFilterLogs(ctx context.Context, id rpc.ID) ([]*types.Log, error) { api.filtersMu.Lock() f, found := api.filters[id] @@ -410,8 +399,6 @@ func (api *PublicFilterAPI) GetFilterLogs(ctx context.Context, id rpc.ID) ([]*ty // // For pending transaction and block filters the result is []common.Hash. // (pending)Log filters return []Log. -// -// https://eth.wiki/json-rpc/API#eth_getfilterchanges func (api *PublicFilterAPI) GetFilterChanges(id rpc.ID) (interface{}, error) { api.filtersMu.Lock() defer api.filtersMu.Unlock() diff --git a/eth/filters/bench_test.go b/eth/filters/bench_test.go index 9632f4195f4cf..694d737350288 100644 --- a/eth/filters/bench_test.go +++ b/eth/filters/bench_test.go @@ -93,9 +93,9 @@ func benchmarkBloomBits(b *testing.B, sectionSize uint64) { var header *types.Header for i := sectionIdx * sectionSize; i < (sectionIdx+1)*sectionSize; i++ { hash := rawdb.ReadCanonicalHash(db, i) - header = rawdb.ReadHeader(db, hash, i) - if header == nil { + if header = rawdb.ReadHeader(db, hash, i); header == nil { b.Fatalf("Error creating bloomBits data") + return } bc.AddBloom(uint(i-sectionIdx*sectionSize), header.Bloom) } @@ -144,9 +144,9 @@ func benchmarkBloomBits(b *testing.B, sectionSize uint64) { db.Close() } -var bloomBitsPrefix = []byte("bloomBits-") - +//nolint:unused func clearBloomBits(db ethdb.Database) { + var bloomBitsPrefix = []byte("bloomBits-") fmt.Println("Clearing bloombits data...") it := db.NewIterator(bloomBitsPrefix, nil) for it.Next() { diff --git a/eth/filters/filter.go b/eth/filters/filter.go index f64e84abb86c9..9ff7ab7f55e1e 100644 --- a/eth/filters/filter.go +++ b/eth/filters/filter.go @@ -36,6 +36,7 @@ type Backend interface { HeaderByHash(ctx context.Context, blockHash common.Hash) (*types.Header, error) GetReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error) GetLogs(ctx context.Context, blockHash common.Hash) ([][]*types.Log, error) + PendingBlockAndReceipts() (*types.Block, types.Receipts) SubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscription SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription @@ -128,26 +129,35 @@ func (f *Filter) Logs(ctx context.Context) ([]*types.Log, error) { } return f.blockLogs(ctx, header) } + // Short-cut if all we care about is pending logs + if f.begin == rpc.PendingBlockNumber.Int64() { + if f.end != rpc.PendingBlockNumber.Int64() { + return nil, errors.New("invalid block range") + } + return f.pendingLogs() + } // Figure out the limits of the filter range header, _ := f.backend.HeaderByNumber(ctx, rpc.LatestBlockNumber) if header == nil { return nil, nil } - head := header.Number.Uint64() - - if f.begin == -1 { + var ( + head = header.Number.Uint64() + end = uint64(f.end) + pending = f.end == rpc.PendingBlockNumber.Int64() + ) + if f.begin == rpc.LatestBlockNumber.Int64() { f.begin = int64(head) } - end := uint64(f.end) - if f.end == -1 { + if f.end == rpc.LatestBlockNumber.Int64() || f.end == rpc.PendingBlockNumber.Int64() { end = head } // Gather all indexed logs, and finish with non indexed ones var ( - logs []*types.Log - err error + logs []*types.Log + err error + size, sections = f.backend.BloomStatus() ) - size, sections := f.backend.BloomStatus() if indexed := sections * size; indexed > uint64(f.begin) { if indexed > end { logs, err = f.indexedLogs(ctx, end) @@ -160,6 +170,13 @@ func (f *Filter) Logs(ctx context.Context) ([]*types.Log, error) { } rest, err := f.unindexedLogs(ctx, end) logs = append(logs, rest...) + if pending { + pendingLogs, err := f.pendingLogs() + if err != nil { + return nil, err + } + logs = append(logs, pendingLogs...) + } return logs, err } @@ -272,6 +289,19 @@ func (f *Filter) checkMatches(ctx context.Context, header *types.Header) (logs [ return nil, nil } +// pendingLogs returns the logs matching the filter criteria within the pending block. +func (f *Filter) pendingLogs() ([]*types.Log, error) { + block, receipts := f.backend.PendingBlockAndReceipts() + if bloomFilter(block.Bloom(), f.addresses, f.topics) { + var unfiltered []*types.Log + for _, r := range receipts { + unfiltered = append(unfiltered, r.Logs...) + } + return filterLogs(unfiltered, nil, nil, f.addresses, f.topics), nil + } + return nil, nil +} + func includes(addresses []common.Address, a common.Address) bool { for _, addr := range addresses { if addr == a { diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go index 87971d5a97c94..629e054160535 100644 --- a/eth/filters/filter_system_test.go +++ b/eth/filters/filter_system_test.go @@ -44,7 +44,6 @@ var ( ) type testBackend struct { - mux *event.TypeMux db ethdb.Database sections uint64 txFeed event.Feed @@ -106,6 +105,10 @@ func (b *testBackend) GetLogs(ctx context.Context, hash common.Hash) ([][]*types return logs, nil } +func (b *testBackend) PendingBlockAndReceipts() (*types.Block, types.Receipts) { + return nil, nil +} + func (b *testBackend) SubscribeNewTxsEvent(ch chan<- core.NewTxsEvent) event.Subscription { return b.txFeed.Subscribe(ch) } @@ -592,7 +595,7 @@ func TestPendingLogsSubscription(t *testing.T) { // (some) events are posted. for i := range testCases { testCases[i].c = make(chan []*types.Log) - testCases[i].err = make(chan error) + testCases[i].err = make(chan error, 1) var err error testCases[i].sub, err = api.events.SubscribeLogs(testCases[i].crit, testCases[i].c) diff --git a/eth/gasprice/gasprice_test.go b/eth/gasprice/gasprice_test.go index c0d3c6b6038e9..95a908fc1e9c3 100644 --- a/eth/gasprice/gasprice_test.go +++ b/eth/gasprice/gasprice_test.go @@ -108,6 +108,7 @@ func newTestBackend(t *testing.T, londonBlock *big.Int, pending bool) *testBacke ) config.LondonBlock = londonBlock config.ArrowGlacierBlock = londonBlock + config.GrayGlacierBlock = londonBlock engine := ethash.NewFaker() db := rawdb.NewMemoryDatabase() genesis, err := gspec.Commit(db) diff --git a/eth/peer.go b/eth/peer.go index 024a6e619371f..55e5f0046206a 100644 --- a/eth/peer.go +++ b/eth/peer.go @@ -34,8 +34,7 @@ type ethPeerInfo struct { // ethPeer is a wrapper around eth.Peer to maintain a few extra metadata. type ethPeer struct { *eth.Peer - snapExt *snapPeer // Satellite `snap` connection - snapWait chan struct{} // Notification channel for snap connections + snapExt *snapPeer // Satellite `snap` connection } // info gathers and returns some `eth` protocol metadata known about a peer. diff --git a/eth/protocols/eth/handler.go b/eth/protocols/eth/handler.go index 81d45d8b8fcf0..3a0b21c30bdb2 100644 --- a/eth/protocols/eth/handler.go +++ b/eth/protocols/eth/handler.go @@ -181,6 +181,21 @@ var eth66 = map[uint64]msgHandler{ PooledTransactionsMsg: handlePooledTransactions66, } +var eth67 = map[uint64]msgHandler{ + NewBlockHashesMsg: handleNewBlockhashes, + NewBlockMsg: handleNewBlock, + TransactionsMsg: handleTransactions, + NewPooledTransactionHashesMsg: handleNewPooledTransactionHashes, + GetBlockHeadersMsg: handleGetBlockHeaders66, + BlockHeadersMsg: handleBlockHeaders66, + GetBlockBodiesMsg: handleGetBlockBodies66, + BlockBodiesMsg: handleBlockBodies66, + GetReceiptsMsg: handleGetReceipts66, + ReceiptsMsg: handleReceipts66, + GetPooledTransactionsMsg: handleGetPooledTransactions66, + PooledTransactionsMsg: handlePooledTransactions66, +} + // handleMessage is invoked whenever an inbound message is received from a remote // peer. The remote connection is torn down upon returning any error. func handleMessage(backend Backend, peer *Peer) error { @@ -195,9 +210,9 @@ func handleMessage(backend Backend, peer *Peer) error { defer msg.Discard() var handlers = eth66 - //if peer.Version() >= ETH67 { // Left in as a sample when new protocol is added - // handlers = eth67 - //} + if peer.Version() >= ETH67 { + handlers = eth67 + } // Track the amount of time it takes to serve the request and run the handler if metrics.Enabled { diff --git a/eth/protocols/eth/protocol.go b/eth/protocols/eth/protocol.go index 24b65f01dd96a..f6fac42780805 100644 --- a/eth/protocols/eth/protocol.go +++ b/eth/protocols/eth/protocol.go @@ -31,6 +31,7 @@ import ( // Constants to match up protocol versions and messages const ( ETH66 = 66 + ETH67 = 67 ) // ProtocolName is the official short name of the `eth` protocol used during @@ -39,11 +40,11 @@ const ProtocolName = "eth" // ProtocolVersions are the supported versions of the `eth` protocol (first // is primary). -var ProtocolVersions = []uint{ETH66} +var ProtocolVersions = []uint{ETH67, ETH66} // protocolLengths are the number of implemented message corresponding to // different protocol versions. -var protocolLengths = map[uint]uint64{ETH66: 17} +var protocolLengths = map[uint]uint64{ETH67: 17, ETH66: 17} // maxMessageSize is the maximum cap on the size of a protocol message. const maxMessageSize = 10 * 1024 * 1024 diff --git a/eth/protocols/snap/handler.go b/eth/protocols/snap/handler.go index 23638ef888849..7ecf041e9a54c 100644 --- a/eth/protocols/snap/handler.go +++ b/eth/protocols/snap/handler.go @@ -285,7 +285,7 @@ func ServiceGetAccountRangeQuery(chain *core.BlockChain, req *GetAccountRangePac req.Bytes = softResponseLimit } // Retrieve the requested state and bail out if non existent - tr, err := trie.New(req.Root, chain.StateCache().TrieDB()) + tr, err := trie.New(common.Hash{}, req.Root, chain.StateCache().TrieDB()) if err != nil { return nil, nil } @@ -415,7 +415,7 @@ func ServiceGetStorageRangesQuery(chain *core.BlockChain, req *GetStorageRangesP if origin != (common.Hash{}) || (abort && len(storage) > 0) { // Request started at a non-zero hash or was capped prematurely, add // the endpoint Merkle proofs - accTrie, err := trie.New(req.Root, chain.StateCache().TrieDB()) + accTrie, err := trie.New(common.Hash{}, req.Root, chain.StateCache().TrieDB()) if err != nil { return nil, nil } @@ -423,7 +423,7 @@ func ServiceGetStorageRangesQuery(chain *core.BlockChain, req *GetStorageRangesP if err := rlp.DecodeBytes(accTrie.Get(account[:]), &acc); err != nil { return nil, nil } - stTrie, err := trie.New(acc.Root, chain.StateCache().TrieDB()) + stTrie, err := trie.New(account, acc.Root, chain.StateCache().TrieDB()) if err != nil { return nil, nil } @@ -489,7 +489,7 @@ func ServiceGetTrieNodesQuery(chain *core.BlockChain, req *GetTrieNodesPacket, s // Make sure we have the state associated with the request triedb := chain.StateCache().TrieDB() - accTrie, err := trie.NewSecure(req.Root, triedb) + accTrie, err := trie.NewSecure(common.Hash{}, req.Root, triedb) if err != nil { // We don't have the requested state available, bail out return nil, nil @@ -531,7 +531,7 @@ func ServiceGetTrieNodesQuery(chain *core.BlockChain, req *GetTrieNodesPacket, s if err != nil || account == nil { break } - stTrie, err := trie.NewSecure(common.BytesToHash(account.Root), triedb) + stTrie, err := trie.NewSecure(common.BytesToHash(pathset[0]), common.BytesToHash(account.Root), triedb) loads++ // always account database reads, even for failures if err != nil { break diff --git a/eth/protocols/snap/sort_test.go b/eth/protocols/snap/sort_test.go index c625be09ea549..49730c886e7f5 100644 --- a/eth/protocols/snap/sort_test.go +++ b/eth/protocols/snap/sort_test.go @@ -67,10 +67,9 @@ func TestRequestSorting(t *testing.T) { "0x01234567890123456789012345678901012345678901234567890123456789010", "0x01234567890123456789012345678901012345678901234567890123456789011", } { - sp, tnps, hash := f(x) + sp, _, hash := f(x) hashes = append(hashes, hash) paths = append(paths, sp) - pathsets = append(pathsets, tnps) } _, paths, pathsets = sortByAccountPath(hashes, paths) { diff --git a/eth/protocols/snap/sync.go b/eth/protocols/snap/sync.go index 415253c839be2..d68e728ff211b 100644 --- a/eth/protocols/snap/sync.go +++ b/eth/protocols/snap/sync.go @@ -712,7 +712,7 @@ func (s *Syncer) loadSyncStatus() { } task.genTrie = trie.NewStackTrie(task.genBatch) - for _, subtasks := range task.SubTasks { + for accountHash, subtasks := range task.SubTasks { for _, subtask := range subtasks { subtask.genBatch = ethdb.HookedBatch{ Batch: s.db.NewBatch(), @@ -720,7 +720,7 @@ func (s *Syncer) loadSyncStatus() { s.storageBytes += common.StorageSize(len(key) + len(value)) }, } - subtask.genTrie = trie.NewStackTrie(subtask.genBatch) + subtask.genTrie = trie.NewStackTrieWithOwner(subtask.genBatch, accountHash) } } } @@ -1971,7 +1971,7 @@ func (s *Syncer) processStorageResponse(res *storageResponse) { Last: r.End(), root: acc.Root, genBatch: batch, - genTrie: trie.NewStackTrie(batch), + genTrie: trie.NewStackTrieWithOwner(batch, account), }) for r.Next() { batch := ethdb.HookedBatch{ @@ -1985,7 +1985,7 @@ func (s *Syncer) processStorageResponse(res *storageResponse) { Last: r.End(), root: acc.Root, genBatch: batch, - genTrie: trie.NewStackTrie(batch), + genTrie: trie.NewStackTrieWithOwner(batch, account), }) } for _, task := range tasks { @@ -2030,7 +2030,7 @@ func (s *Syncer) processStorageResponse(res *storageResponse) { slots += len(res.hashes[i]) if i < len(res.hashes)-1 || res.subTask == nil { - tr := trie.NewStackTrie(batch) + tr := trie.NewStackTrieWithOwner(batch, account) for j := 0; j < len(res.hashes[i]); j++ { tr.Update(res.hashes[i][j][:], res.slots[i][j]) } @@ -2781,7 +2781,7 @@ func (s *Syncer) onHealState(paths [][]byte, value []byte) error { if len(paths) == 1 { var account types.StateAccount if err := rlp.DecodeBytes(value, &account); err != nil { - return nil + return nil // Returning the error here would drop the remote peer } blob := snapshot.SlimAccountRLP(account.Nonce, account.Balance, account.Root, account.CodeHash) rawdb.WriteAccountSnapshot(s.stateWriter, common.BytesToHash(paths[0]), blob) diff --git a/eth/protocols/snap/sync_test.go b/eth/protocols/snap/sync_test.go index e727544fa4500..85e4dc5e4f835 100644 --- a/eth/protocols/snap/sync_test.go +++ b/eth/protocols/snap/sync_test.go @@ -1349,7 +1349,7 @@ func getCodeByHash(hash common.Hash) []byte { // makeAccountTrieNoStorage spits out a trie, along with the leafs func makeAccountTrieNoStorage(n int) (*trie.Trie, entrySlice) { db := trie.NewDatabase(rawdb.NewMemoryDatabase()) - accTrie, _ := trie.New(common.Hash{}, db) + accTrie := trie.NewEmpty(db) var entries entrySlice for i := uint64(1); i <= uint64(n); i++ { value, _ := rlp.EncodeToBytes(&types.StateAccount{ @@ -1376,8 +1376,8 @@ func makeBoundaryAccountTrie(n int) (*trie.Trie, entrySlice) { entries entrySlice boundaries []common.Hash - db = trie.NewDatabase(rawdb.NewMemoryDatabase()) - trie, _ = trie.New(common.Hash{}, db) + db = trie.NewDatabase(rawdb.NewMemoryDatabase()) + trie = trie.NewEmpty(db) ) // Initialize boundaries var next common.Hash @@ -1429,7 +1429,7 @@ func makeBoundaryAccountTrie(n int) (*trie.Trie, entrySlice) { func makeAccountTrieWithStorageWithUniqueStorage(accounts, slots int, code bool) (*trie.Trie, entrySlice, map[common.Hash]*trie.Trie, map[common.Hash]entrySlice) { var ( db = trie.NewDatabase(rawdb.NewMemoryDatabase()) - accTrie, _ = trie.New(common.Hash{}, db) + accTrie = trie.NewEmpty(db) entries entrySlice storageTries = make(map[common.Hash]*trie.Trie) storageEntries = make(map[common.Hash]entrySlice) @@ -1442,7 +1442,7 @@ func makeAccountTrieWithStorageWithUniqueStorage(accounts, slots int, code bool) codehash = getCodeHash(i) } // Create a storage trie - stTrie, stEntries := makeStorageTrieWithSeed(uint64(slots), i, db) + stTrie, stEntries := makeStorageTrieWithSeed(common.BytesToHash(key), uint64(slots), i, db) stRoot := stTrie.Hash() stTrie.Commit(nil) value, _ := rlp.EncodeToBytes(&types.StateAccount{ @@ -1468,23 +1468,11 @@ func makeAccountTrieWithStorageWithUniqueStorage(accounts, slots int, code bool) func makeAccountTrieWithStorage(accounts, slots int, code, boundary bool) (*trie.Trie, entrySlice, map[common.Hash]*trie.Trie, map[common.Hash]entrySlice) { var ( db = trie.NewDatabase(rawdb.NewMemoryDatabase()) - accTrie, _ = trie.New(common.Hash{}, db) + accTrie = trie.NewEmpty(db) entries entrySlice storageTries = make(map[common.Hash]*trie.Trie) storageEntries = make(map[common.Hash]entrySlice) ) - // Make a storage trie which we reuse for the whole lot - var ( - stTrie *trie.Trie - stEntries entrySlice - ) - if boundary { - stTrie, stEntries = makeBoundaryStorageTrie(slots, db) - } else { - stTrie, stEntries = makeStorageTrieWithSeed(uint64(slots), 0, db) - } - stRoot := stTrie.Hash() - // Create n accounts in the trie for i := uint64(1); i <= uint64(accounts); i++ { key := key32(i) @@ -1492,6 +1480,19 @@ func makeAccountTrieWithStorage(accounts, slots int, code, boundary bool) (*trie if code { codehash = getCodeHash(i) } + // Make a storage trie + var ( + stTrie *trie.Trie + stEntries entrySlice + ) + if boundary { + stTrie, stEntries = makeBoundaryStorageTrie(common.BytesToHash(key), slots, db) + } else { + stTrie, stEntries = makeStorageTrieWithSeed(common.BytesToHash(key), uint64(slots), 0, db) + } + stRoot := stTrie.Hash() + stTrie.Commit(nil) + value, _ := rlp.EncodeToBytes(&types.StateAccount{ Nonce: i, Balance: big.NewInt(int64(i)), @@ -1506,7 +1507,6 @@ func makeAccountTrieWithStorage(accounts, slots int, code, boundary bool) (*trie storageEntries[common.BytesToHash(key)] = stEntries } sort.Sort(entries) - stTrie.Commit(nil) accTrie.Commit(nil) return accTrie, entries, storageTries, storageEntries } @@ -1514,8 +1514,8 @@ func makeAccountTrieWithStorage(accounts, slots int, code, boundary bool) (*trie // makeStorageTrieWithSeed fills a storage trie with n items, returning the // not-yet-committed trie and the sorted entries. The seeds can be used to ensure // that tries are unique. -func makeStorageTrieWithSeed(n, seed uint64, db *trie.Database) (*trie.Trie, entrySlice) { - trie, _ := trie.New(common.Hash{}, db) +func makeStorageTrieWithSeed(owner common.Hash, n, seed uint64, db *trie.Database) (*trie.Trie, entrySlice) { + trie, _ := trie.New(owner, common.Hash{}, db) var entries entrySlice for i := uint64(1); i <= n; i++ { // store 'x' at slot 'x' @@ -1537,11 +1537,11 @@ func makeStorageTrieWithSeed(n, seed uint64, db *trie.Database) (*trie.Trie, ent // makeBoundaryStorageTrie constructs a storage trie. Instead of filling // storage slots normally, this function will fill a few slots which have // boundary hash. -func makeBoundaryStorageTrie(n int, db *trie.Database) (*trie.Trie, entrySlice) { +func makeBoundaryStorageTrie(owner common.Hash, n int, db *trie.Database) (*trie.Trie, entrySlice) { var ( entries entrySlice boundaries []common.Hash - trie, _ = trie.New(common.Hash{}, db) + trie, _ = trie.New(owner, common.Hash{}, db) ) // Initialize boundaries var next common.Hash @@ -1588,7 +1588,7 @@ func makeBoundaryStorageTrie(n int, db *trie.Database) (*trie.Trie, entrySlice) func verifyTrie(db ethdb.KeyValueStore, root common.Hash, t *testing.T) { t.Helper() triedb := trie.NewDatabase(db) - accTrie, err := trie.New(root, triedb) + accTrie, err := trie.New(common.Hash{}, root, triedb) if err != nil { t.Fatal(err) } @@ -1606,7 +1606,7 @@ func verifyTrie(db ethdb.KeyValueStore, root common.Hash, t *testing.T) { } accounts++ if acc.Root != emptyRoot { - storeTrie, err := trie.NewSecure(acc.Root, triedb) + storeTrie, err := trie.NewSecure(common.BytesToHash(accIt.Key), acc.Root, triedb) if err != nil { t.Fatal(err) } @@ -1661,7 +1661,7 @@ func TestSyncAccountPerformance(t *testing.T) { // Doing so would bring this number down to zero in this artificial testcase, // but only add extra IO for no reason in practice. if have, want := src.nTrienodeRequests, 1; have != want { - fmt.Printf(src.Stats()) + fmt.Print(src.Stats()) t.Errorf("trie node heal requests wrong, want %d, have %d", want, have) } } diff --git a/eth/sync_test.go b/eth/sync_test.go index 929a2a9d181c0..0b9f9e1bbaaf7 100644 --- a/eth/sync_test.go +++ b/eth/sync_test.go @@ -30,6 +30,7 @@ import ( // Tests that snap sync is disabled after a successful sync cycle. func TestSnapSyncDisabling66(t *testing.T) { testSnapSyncDisabling(t, eth.ETH66, snap.SNAP1) } +func TestSnapSyncDisabling67(t *testing.T) { testSnapSyncDisabling(t, eth.ETH67, snap.SNAP1) } // Tests that snap sync gets disabled as soon as a real block is successfully // imported into the blockchain. diff --git a/eth/tracers/api.go b/eth/tracers/api.go index 074f193e64ebe..13b89885138e8 100644 --- a/eth/tracers/api.go +++ b/eth/tracers/api.go @@ -179,6 +179,7 @@ type TraceCallConfig struct { Timeout *string Reexec *uint64 StateOverrides *ethapi.StateOverride + BlockOverrides *ethapi.BlockOverrides } // StdTraceConfig holds extra parameters to standard-json trace functions. @@ -806,7 +807,6 @@ func (api *API) TraceTransaction(ctx context.Context, hash common.Hash, config * // TraceCall lets you trace a given eth_call. It collects the structured logs // created during the execution of EVM if the given transaction was added on // top of the provided block and returns them as a JSON object. -// You can provide -2 as a block number to trace on top of the pending block. func (api *API) TraceCall(ctx context.Context, args ethapi.TransactionArgs, blockNrOrHash rpc.BlockNumberOrHash, config *TraceCallConfig) (interface{}, error) { // Try to retrieve the specified block var ( @@ -816,6 +816,14 @@ func (api *API) TraceCall(ctx context.Context, args ethapi.TransactionArgs, bloc if hash, ok := blockNrOrHash.Hash(); ok { block, err = api.blockByHash(ctx, hash) } else if number, ok := blockNrOrHash.Number(); ok { + if number == rpc.PendingBlockNumber { + // We don't have access to the miner here. For tracing 'future' transactions, + // it can be done with block- and state-overrides instead, which offers + // more flexibility and stability than trying to trace on 'pending', since + // the contents of 'pending' is unstable and probably not a true representation + // of what the next actual block is likely to contain. + return nil, errors.New("tracing on top of pending is not supported") + } block, err = api.blockByNumber(ctx, number) } else { return nil, errors.New("invalid arguments; neither block nor hash specified") @@ -832,18 +840,19 @@ func (api *API) TraceCall(ctx context.Context, args ethapi.TransactionArgs, bloc if err != nil { return nil, err } - // Apply the customized state rules if required. + vmctx := core.NewEVMBlockContext(block.Header(), api.chainContext(ctx), nil) + // Apply the customization rules if required. if config != nil { if err := config.StateOverrides.Apply(statedb); err != nil { return nil, err } + config.BlockOverrides.Apply(&vmctx) } // Execute the trace msg, err := args.ToMessage(api.backend.RPCGasCap(), block.BaseFee()) if err != nil { return nil, err } - vmctx := core.NewEVMBlockContext(block.Header(), api.chainContext(ctx), nil) var traceConfig *TraceConfig if config != nil { diff --git a/eth/tracers/api_test.go b/eth/tracers/api_test.go index af41f05d212b8..bc12b9275160e 100644 --- a/eth/tracers/api_test.go +++ b/eth/tracers/api_test.go @@ -196,13 +196,12 @@ func TestTraceCall(t *testing.T) { tx, _ := types.SignTx(types.NewTransaction(uint64(i), accounts[1].addr, big.NewInt(1000), params.TxGas, b.BaseFee(), nil), signer, accounts[0].key) b.AddTx(tx) })) - var testSuite = []struct { blockNumber rpc.BlockNumber call ethapi.TransactionArgs config *TraceCallConfig expectErr error - expect interface{} + expect string }{ // Standard JSON trace upon the genesis, plain transfer. { @@ -214,12 +213,7 @@ func TestTraceCall(t *testing.T) { }, config: nil, expectErr: nil, - expect: &logger.ExecutionResult{ - Gas: params.TxGas, - Failed: false, - ReturnValue: "", - StructLogs: []logger.StructLogRes{}, - }, + expect: `{"gas":21000,"failed":false,"returnValue":"","structLogs":[]}`, }, // Standard JSON trace upon the head, plain transfer. { @@ -231,12 +225,7 @@ func TestTraceCall(t *testing.T) { }, config: nil, expectErr: nil, - expect: &logger.ExecutionResult{ - Gas: params.TxGas, - Failed: false, - ReturnValue: "", - StructLogs: []logger.StructLogRes{}, - }, + expect: `{"gas":21000,"failed":false,"returnValue":"","structLogs":[]}`, }, // Standard JSON trace upon the non-existent block, error expects { @@ -248,7 +237,7 @@ func TestTraceCall(t *testing.T) { }, config: nil, expectErr: fmt.Errorf("block #%d not found", genBlocks+1), - expect: nil, + //expect: nil, }, // Standard JSON trace upon the latest block { @@ -260,14 +249,9 @@ func TestTraceCall(t *testing.T) { }, config: nil, expectErr: nil, - expect: &logger.ExecutionResult{ - Gas: params.TxGas, - Failed: false, - ReturnValue: "", - StructLogs: []logger.StructLogRes{}, - }, + expect: `{"gas":21000,"failed":false,"returnValue":"","structLogs":[]}`, }, - // Standard JSON trace upon the pending block + // Tracing on 'pending' should fail: { blockNumber: rpc.PendingBlockNumber, call: ethapi.TransactionArgs{ @@ -276,36 +260,48 @@ func TestTraceCall(t *testing.T) { Value: (*hexutil.Big)(big.NewInt(1000)), }, config: nil, - expectErr: nil, - expect: &logger.ExecutionResult{ - Gas: params.TxGas, - Failed: false, - ReturnValue: "", - StructLogs: []logger.StructLogRes{}, + expectErr: errors.New("tracing on top of pending is not supported"), + }, + { + blockNumber: rpc.LatestBlockNumber, + call: ethapi.TransactionArgs{ + From: &accounts[0].addr, + Input: &hexutil.Bytes{0x43}, // blocknumber + }, + config: &TraceCallConfig{ + BlockOverrides: ðapi.BlockOverrides{Number: (*hexutil.Big)(big.NewInt(0x1337))}, }, + expectErr: nil, + expect: ` {"gas":53018,"failed":false,"returnValue":"","structLogs":[ + {"pc":0,"op":"NUMBER","gas":24946984,"gasCost":2,"depth":1,"stack":[]}, + {"pc":1,"op":"STOP","gas":24946982,"gasCost":0,"depth":1,"stack":["0x1337"]}]}`, }, } - for _, testspec := range testSuite { + for i, testspec := range testSuite { result, err := api.TraceCall(context.Background(), testspec.call, rpc.BlockNumberOrHash{BlockNumber: &testspec.blockNumber}, testspec.config) if testspec.expectErr != nil { if err == nil { - t.Errorf("Expect error %v, get nothing", testspec.expectErr) + t.Errorf("test %d: expect error %v, got nothing", i, testspec.expectErr) continue } if !reflect.DeepEqual(err, testspec.expectErr) { - t.Errorf("Error mismatch, want %v, get %v", testspec.expectErr, err) + t.Errorf("test %d: error mismatch, want %v, git %v", i, testspec.expectErr, err) } } else { if err != nil { - t.Errorf("Expect no error, get %v", err) + t.Errorf("test %d: expect no error, got %v", i, err) continue } var have *logger.ExecutionResult if err := json.Unmarshal(result.(json.RawMessage), &have); err != nil { - t.Errorf("failed to unmarshal result %v", err) + t.Errorf("test %d: failed to unmarshal result %v", i, err) + } + var want *logger.ExecutionResult + if err := json.Unmarshal([]byte(testspec.expect), &want); err != nil { + t.Errorf("test %d: failed to unmarshal result %v", i, err) } - if !reflect.DeepEqual(have, testspec.expect) { - t.Errorf("Result mismatch, want %v, get %v", testspec.expect, have) + if !reflect.DeepEqual(have, want) { + t.Errorf("test %d: result mismatch, want %v, got %v", i, testspec.expect, string(result.(json.RawMessage))) } } } @@ -446,7 +442,7 @@ func TestTracingWithOverrides(t *testing.T) { type res struct { Gas int Failed bool - returnValue string + ReturnValue string } var testSuite = []struct { blockNumber rpc.BlockNumber @@ -457,7 +453,7 @@ func TestTracingWithOverrides(t *testing.T) { }{ // Call which can only succeed if state is state overridden { - blockNumber: rpc.PendingBlockNumber, + blockNumber: rpc.LatestBlockNumber, call: ethapi.TransactionArgs{ From: &randomAccounts[0].addr, To: &randomAccounts[1].addr, @@ -472,7 +468,7 @@ func TestTracingWithOverrides(t *testing.T) { }, // Invalid call without state overriding { - blockNumber: rpc.PendingBlockNumber, + blockNumber: rpc.LatestBlockNumber, call: ethapi.TransactionArgs{ From: &randomAccounts[0].addr, To: &randomAccounts[1].addr, @@ -498,7 +494,7 @@ func TestTracingWithOverrides(t *testing.T) { // } // } { - blockNumber: rpc.PendingBlockNumber, + blockNumber: rpc.LatestBlockNumber, call: ethapi.TransactionArgs{ From: &randomAccounts[0].addr, To: &randomAccounts[2].addr, @@ -515,6 +511,39 @@ func TestTracingWithOverrides(t *testing.T) { }, want: `{"gas":23347,"failed":false,"returnValue":"000000000000000000000000000000000000000000000000000000000000007b"}`, }, + { // Override blocknumber + blockNumber: rpc.LatestBlockNumber, + call: ethapi.TransactionArgs{ + From: &accounts[0].addr, + // BLOCKNUMBER PUSH1 MSTORE + Input: newRPCBytes(common.Hex2Bytes("4360005260206000f3")), + //&hexutil.Bytes{0x43}, // blocknumber + }, + config: &TraceCallConfig{ + BlockOverrides: ðapi.BlockOverrides{Number: (*hexutil.Big)(big.NewInt(0x1337))}, + }, + want: `{"gas":59537,"failed":false,"returnValue":"0000000000000000000000000000000000000000000000000000000000001337"}`, + }, + { // Override blocknumber, and query a blockhash + blockNumber: rpc.LatestBlockNumber, + call: ethapi.TransactionArgs{ + From: &accounts[0].addr, + Input: &hexutil.Bytes{ + 0x60, 0x00, 0x40, // BLOCKHASH(0) + 0x60, 0x00, 0x52, // STORE memory offset 0 + 0x61, 0x13, 0x36, 0x40, // BLOCKHASH(0x1336) + 0x60, 0x20, 0x52, // STORE memory offset 32 + 0x61, 0x13, 0x37, 0x40, // BLOCKHASH(0x1337) + 0x60, 0x40, 0x52, // STORE memory offset 64 + 0x60, 0x60, 0x60, 0x00, 0xf3, // RETURN (0-96) + + }, // blocknumber + }, + config: &TraceCallConfig{ + BlockOverrides: ðapi.BlockOverrides{Number: (*hexutil.Big)(big.NewInt(0x1337))}, + }, + want: `{"gas":72666,"failed":false,"returnValue":"000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"}`, + }, } for i, tc := range testSuite { result, err := api.TraceCall(context.Background(), tc.call, rpc.BlockNumberOrHash{BlockNumber: &tc.blockNumber}, tc.config) diff --git a/eth/tracers/js/internal/tracers/4byte_tracer_legacy.js b/eth/tracers/js/internal/tracers/4byte_tracer_legacy.js index 462b4ad4cb550..e4714b8bfb762 100644 --- a/eth/tracers/js/internal/tracers/4byte_tracer_legacy.js +++ b/eth/tracers/js/internal/tracers/4byte_tracer_legacy.js @@ -46,7 +46,7 @@ return false; }, - // store save the given indentifier and datasize. + // store save the given identifier and datasize. store: function(id, size){ var key = "" + toHex(id) + "-" + size; this.ids[key] = this.ids[key] + 1 || 1; diff --git a/eth/tracers/tracers_test.go b/eth/tracers/tracers_test.go index ce9289dd756b2..12e01abae4034 100644 --- a/eth/tracers/tracers_test.go +++ b/eth/tracers/tracers_test.go @@ -21,7 +21,6 @@ import ( "testing" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" @@ -32,20 +31,6 @@ import ( "github.com/ethereum/go-ethereum/tests" ) -// callTrace is the result of a callTracer run. -type callTrace struct { - Type string `json:"type"` - From common.Address `json:"from"` - To common.Address `json:"to"` - Input hexutil.Bytes `json:"input"` - Output hexutil.Bytes `json:"output"` - Gas *hexutil.Uint64 `json:"gas,omitempty"` - GasUsed *hexutil.Uint64 `json:"gasUsed,omitempty"` - Value *hexutil.Big `json:"value,omitempty"` - Error string `json:"error,omitempty"` - Calls []callTrace `json:"calls,omitempty"` -} - func BenchmarkTransactionTrace(b *testing.B) { key, _ := crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") from := crypto.PubkeyToAddress(key.PublicKey) diff --git a/go.mod b/go.mod index de9d4e4dfde0f..dd24d7bda0239 100644 --- a/go.mod +++ b/go.mod @@ -1,10 +1,9 @@ module github.com/ethereum/go-ethereum -go 1.16 +go 1.17 require ( github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0 - github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 // indirect github.com/VictoriaMetrics/fastcache v1.6.0 github.com/aws/aws-sdk-go-v2 v1.2.0 github.com/aws/aws-sdk-go-v2/config v1.1.1 @@ -16,7 +15,6 @@ require ( github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f github.com/davecgh/go-spew v1.1.1 github.com/deckarep/golang-set v1.8.0 - github.com/deepmap/oapi-codegen v1.8.2 // indirect github.com/docker/docker v1.6.2 github.com/dop251/goja v0.0.0-20220405120441-9037c2b61cbf github.com/edsrzf/mmap-go v1.0.0 @@ -70,6 +68,7 @@ require ( github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 github.com/stretchr/testify v1.7.0 + github.com/supranational/blst v0.3.8-0.20220526154634-513d2456b344 github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 github.com/thoas/go-funk v0.9.2 github.com/tklauser/go-sysconf v0.3.5 // indirect @@ -78,9 +77,65 @@ require ( golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c golang.org/x/sys v0.0.0-20211019181941-9d821ace8654 + golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 golang.org/x/text v0.3.7 golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023 gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce gopkg.in/urfave/cli.v1 v1.20.0 ) + +require ( + github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.3 // indirect + github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.0.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.0.2 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.1.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.1.1 // indirect + github.com/aws/smithy-go v1.1.0 // indirect + github.com/cespare/xxhash/v2 v2.1.1 // indirect + github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect + github.com/deepmap/oapi-codegen v1.8.2 // indirect + github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91 // indirect + github.com/garslo/gogen v0.0.0-20170306192744-1d203ffc1f61 // indirect + github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect + github.com/gogo/protobuf v1.3.1 // indirect + github.com/ipfs/bbloom v0.0.4 // indirect + github.com/ipfs/go-datastore v0.4.2 // indirect + github.com/ipfs/go-ipfs-util v0.0.1 // indirect + github.com/ipfs/go-log v0.0.1 // indirect + github.com/ipfs/go-metrics-interface v0.0.1 // indirect + github.com/jackc/chunkreader/v2 v2.0.1 // indirect + github.com/jackc/pgio v1.0.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgproto3/v2 v2.1.1 // indirect + github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect + github.com/jackc/pgtype v1.8.1 // indirect + github.com/jackc/puddle v1.1.3 // indirect + github.com/jbenet/goprocess v0.0.0-20160826012719-b497e2f366b8 // indirect + github.com/mattn/go-runewidth v0.0.9 // indirect + github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 // indirect + github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771 // indirect + github.com/mitchellh/mapstructure v1.4.1 // indirect + github.com/mitchellh/pointerstructure v1.2.0 // indirect + github.com/mr-tron/base58 v1.1.3 // indirect + github.com/multiformats/go-base32 v0.0.3 // indirect + github.com/multiformats/go-base36 v0.1.0 // indirect + github.com/multiformats/go-multibase v0.0.3 // indirect + github.com/multiformats/go-varint v0.0.5 // indirect + github.com/opentracing/opentracing-go v1.1.0 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/spaolacci/murmur3 v1.1.0 // indirect + github.com/stretchr/objx v0.2.0 // indirect + github.com/tklauser/numcpus v0.2.2 // indirect + github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc // indirect + go.uber.org/atomic v1.6.0 // indirect + golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57 // indirect + golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f // indirect + golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect + google.golang.org/protobuf v1.23.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect +) diff --git a/go.sum b/go.sum index ee7f3d6ea0b6b..3b29fcb7ce704 100644 --- a/go.sum +++ b/go.sum @@ -570,6 +570,8 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/supranational/blst v0.3.8-0.20220526154634-513d2456b344 h1:m+8fKfQwCAy1QjzINvKe/pYtLjo2dl59x2w9YSEJxuY= +github.com/supranational/blst v0.3.8-0.20220526154634-513d2456b344/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/thoas/go-funk v0.9.2 h1:oKlNYv0AY5nyf9g+/GhMgS/UO2ces0QRdPKwkhY3VCk= diff --git a/graphql/graphql_test.go b/graphql/graphql_test.go index 2768026b5197e..4b7f7bf960216 100644 --- a/graphql/graphql_test.go +++ b/graphql/graphql_test.go @@ -45,10 +45,10 @@ func TestBuildSchema(t *testing.T) { conf := node.DefaultConfig conf.DataDir = ddir stack, err := node.New(&conf) - defer stack.Close() if err != nil { t.Fatalf("could not create new node: %v", err) } + defer stack.Close() // Make sure the schema can be parsed and matched up to the object model. if err := newHandler(stack, nil, []string{}, []string{}); err != nil { t.Errorf("Could not construct GraphQL handler: %v", err) diff --git a/internal/build/gotool.go b/internal/build/gotool.go index e644b5f69526d..08c8b2ef05fe8 100644 --- a/internal/build/gotool.go +++ b/internal/build/gotool.go @@ -85,7 +85,7 @@ func (g *GoToolchain) goTool(command string, args ...string) *exec.Cmd { if g.Root == "" { g.Root = runtime.GOROOT() } - tool := exec.Command(filepath.Join(g.Root, "bin", "go"), command) + tool := exec.Command(filepath.Join(g.Root, "bin", "go"), command) // nolint: gosec tool.Args = append(tool.Args, args...) tool.Env = append(tool.Env, "GOROOT="+g.Root) diff --git a/internal/debug/flags.go b/internal/debug/flags.go index 3aa990adfb3ab..d1bd31abeed65 100644 --- a/internal/debug/flags.go +++ b/internal/debug/flags.go @@ -20,7 +20,7 @@ import ( "fmt" "io" "net/http" - _ "net/http/pprof" + _ "net/http/pprof" // nolint: gosec "os" "runtime" diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 1b7c177867477..4aa37a8dc4c39 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -505,7 +505,7 @@ func (s *PrivateAccountAPI) SignTransaction(ctx context.Context, args Transactio } // Sign calculates an Ethereum ECDSA signature for: -// keccack256("\x19Ethereum Signed Message:\n" + len(message) + message)) +// keccak256("\x19Ethereum Signed Message:\n" + len(message) + message)) // // Note, the produced signature conforms to the secp256k1 curve R, S and V values, // where the V value will be 27 or 28 for legacy reasons. @@ -888,6 +888,41 @@ func (diff *StateOverride) Apply(state *state.StateDB) error { return nil } +// BlockOverrides is a set of header fields to override. +type BlockOverrides struct { + Number *hexutil.Big + Difficulty *hexutil.Big + Time *hexutil.Big + GasLimit *hexutil.Uint64 + Coinbase *common.Address + Random *common.Hash +} + +// Apply overrides the given header fields into the given block context. +func (diff *BlockOverrides) Apply(blockCtx *vm.BlockContext) { + if diff == nil { + return + } + if diff.Number != nil { + blockCtx.BlockNumber = diff.Number.ToInt() + } + if diff.Difficulty != nil { + blockCtx.Difficulty = diff.Difficulty.ToInt() + } + if diff.Time != nil { + blockCtx.Time = diff.Time.ToInt() + } + if diff.GasLimit != nil { + blockCtx.GasLimit = uint64(*diff.GasLimit) + } + if diff.Coinbase != nil { + blockCtx.Coinbase = *diff.Coinbase + } + if diff.Random != nil { + blockCtx.Random = diff.Random + } +} + func DoCall(ctx context.Context, b Backend, args TransactionArgs, blockNrOrHash rpc.BlockNumberOrHash, overrides *StateOverride, timeout time.Duration, globalGasCap uint64) (*core.ExecutionResult, error) { defer func(start time.Time) { log.Debug("Executing EVM call finished", "runtime", time.Since(start)) }(time.Now()) @@ -1233,7 +1268,7 @@ type RPCTransaction struct { // newRPCTransaction returns a transaction that will serialize to the RPC // representation, with the given location metadata set (if available). func newRPCTransaction(tx *types.Transaction, blockHash common.Hash, blockNumber uint64, index uint64, baseFee *big.Int, config *params.ChainConfig) *RPCTransaction { - signer := types.MakeSigner(config, big.NewInt(0).SetUint64(blockNumber)) + signer := types.MakeSigner(config, new(big.Int).SetUint64(blockNumber)) from, _ := types.Sender(signer, tx) v, r, s := tx.RawSignatureValues() result := &RPCTransaction{ @@ -1706,7 +1741,7 @@ func (s *PublicTransactionPoolAPI) SendRawTransaction(ctx context.Context, input } // Sign calculates an ECDSA signature for: -// keccack256("\x19Ethereum Signed Message:\n" + len(message) + message). +// keccak256("\x19Ethereum Signed Message:\n" + len(message) + message). // // Note, the produced signature conforms to the secp256k1 curve R, S and V values, // where the V value will be 27 or 28 for legacy reasons. diff --git a/internal/ethapi/backend.go b/internal/ethapi/backend.go index bc60fb2a64f6c..c11d842df7d57 100644 --- a/internal/ethapi/backend.go +++ b/internal/ethapi/backend.go @@ -65,6 +65,7 @@ type Backend interface { BlockByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Block, error) StateAndHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*state.StateDB, *types.Header, error) StateAndHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*state.StateDB, *types.Header, error) + PendingBlockAndReceipts() (*types.Block, types.Receipts) GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) GetTd(ctx context.Context, hash common.Hash) *big.Int GetEVM(ctx context.Context, msg core.Message, state *state.StateDB, header *types.Header, vmConfig *vm.Config) (*vm.EVM, func() error, error) diff --git a/les/client.go b/les/client.go index c3acbc2e4bf8a..b38a0aae74d3f 100644 --- a/les/client.go +++ b/les/client.go @@ -19,6 +19,7 @@ package les import ( "fmt" + "strings" "time" "github.com/ethereum/go-ethereum/accounts" @@ -92,11 +93,17 @@ func New(stack *node.Node, config *ethconfig.Config) (*LightEthereum, error) { if err != nil { return nil, err } - chainConfig, genesisHash, genesisErr := core.SetupGenesisBlockWithOverride(chainDb, config.Genesis, config.OverrideArrowGlacier, config.OverrideTerminalTotalDifficulty) + chainConfig, genesisHash, genesisErr := core.SetupGenesisBlockWithOverride(chainDb, config.Genesis, config.OverrideGrayGlacier, config.OverrideTerminalTotalDifficulty) if _, isCompat := genesisErr.(*params.ConfigCompatError); genesisErr != nil && !isCompat { return nil, genesisErr } - log.Info("Initialised chain configuration", "config", chainConfig) + log.Info("") + log.Info(strings.Repeat("-", 153)) + for _, line := range strings.Split(chainConfig.String(), "\n") { + log.Info(line) + } + log.Info(strings.Repeat("-", 153)) + log.Info("") peers := newServerPeerSet() merger := consensus.NewMerger(chainDb) diff --git a/les/downloader/downloader_test.go b/les/downloader/downloader_test.go index f6510eb412378..b4b12b1b2957b 100644 --- a/les/downloader/downloader_test.go +++ b/les/downloader/downloader_test.go @@ -229,7 +229,7 @@ func (dl *downloadTester) CurrentFastBlock() *types.Block { func (dl *downloadTester) FastSyncCommitHead(hash common.Hash) error { // For now only check that the state trie is correct if block := dl.GetBlockByHash(hash); block != nil { - _, err := trie.NewSecure(block.Root(), trie.NewDatabase(dl.stateDb)) + _, err := trie.NewSecure(common.Hash{}, block.Root(), trie.NewDatabase(dl.stateDb)) return err } return fmt.Errorf("non existent block: %x", hash[:4]) diff --git a/les/downloader/peer.go b/les/downloader/peer.go index 8632948329711..5a92e9cf9b87a 100644 --- a/les/downloader/peer.go +++ b/les/downloader/peer.go @@ -413,7 +413,7 @@ func (ps *peerSet) HeaderIdlePeers() ([]*peerConnection, int) { throughput := func(p *peerConnection) int { return p.rates.Capacity(eth.BlockHeadersMsg, time.Second) } - return ps.idlePeers(eth.ETH66, eth.ETH66, idle, throughput) + return ps.idlePeers(eth.ETH66, eth.ETH67, idle, throughput) } // BodyIdlePeers retrieves a flat list of all the currently body-idle peers within @@ -425,7 +425,7 @@ func (ps *peerSet) BodyIdlePeers() ([]*peerConnection, int) { throughput := func(p *peerConnection) int { return p.rates.Capacity(eth.BlockBodiesMsg, time.Second) } - return ps.idlePeers(eth.ETH66, eth.ETH66, idle, throughput) + return ps.idlePeers(eth.ETH66, eth.ETH67, idle, throughput) } // ReceiptIdlePeers retrieves a flat list of all the currently receipt-idle peers @@ -437,7 +437,7 @@ func (ps *peerSet) ReceiptIdlePeers() ([]*peerConnection, int) { throughput := func(p *peerConnection) int { return p.rates.Capacity(eth.ReceiptsMsg, time.Second) } - return ps.idlePeers(eth.ETH66, eth.ETH66, idle, throughput) + return ps.idlePeers(eth.ETH66, eth.ETH67, idle, throughput) } // NodeDataIdlePeers retrieves a flat list of all the currently node-data-idle @@ -449,7 +449,7 @@ func (ps *peerSet) NodeDataIdlePeers() ([]*peerConnection, int) { throughput := func(p *peerConnection) int { return p.rates.Capacity(eth.NodeDataMsg, time.Second) } - return ps.idlePeers(eth.ETH66, eth.ETH66, idle, throughput) + return ps.idlePeers(eth.ETH66, eth.ETH67, idle, throughput) } // idlePeers retrieves a flat list of all currently idle peers satisfying the diff --git a/les/downloader/statesync.go b/les/downloader/statesync.go index 2b3278822996b..fd24c5150b3cd 100644 --- a/les/downloader/statesync.go +++ b/les/downloader/statesync.go @@ -608,7 +608,7 @@ func (s *stateSync) updateStats(written, duplicate, unexpected int, duration tim if written > 0 || duplicate > 0 || unexpected > 0 { log.Info("Imported new state entries", "count", written, "elapsed", common.PrettyDuration(duration), "processed", s.d.syncStatsState.processed, "pending", s.d.syncStatsState.pending, "trieretry", len(s.trieTasks), "coderetry", len(s.codeTasks), "duplicate", s.d.syncStatsState.duplicate, "unexpected", s.d.syncStatsState.unexpected) } - if written > 0 { - //rawdb.WriteFastTrieProgress(s.d.stateDB, s.d.syncStatsState.processed) - } + //if written > 0 { + //rawdb.WriteFastTrieProgress(s.d.stateDB, s.d.syncStatsState.processed) + //} } diff --git a/les/fetcher_test.go b/les/fetcher_test.go index 28db3b8913aca..6a17e73757a54 100644 --- a/les/fetcher_test.go +++ b/les/fetcher_test.go @@ -160,7 +160,6 @@ func testTrustedAnnouncement(t *testing.T, protocol int) { nodes []*enode.Node ids []string cpeers []*clientPeer - speers []*serverPeer config = light.TestServerIndexerConfig waitIndexers = func(cIndexer, bIndexer, btIndexer *core.ChainIndexer) { @@ -213,12 +212,11 @@ func testTrustedAnnouncement(t *testing.T, protocol int) { // Connect all server instances. for i := 0; i < len(servers); i++ { - sp, cp, err := connect(servers[i].handler, nodes[i].ID(), c.handler, protocol, true) + _, cp, err := connect(servers[i].handler, nodes[i].ID(), c.handler, protocol, true) if err != nil { t.Fatalf("connect server and client failed, err %s", err) } cpeers = append(cpeers, cp) - speers = append(speers, sp) } newHead := make(chan *types.Header, 1) c.handler.fetcher.newHeadHook = func(header *types.Header) { newHead <- header } diff --git a/les/flowcontrol/manager.go b/les/flowcontrol/manager.go index c9e681c1440a8..4ffbee58f0d0d 100644 --- a/les/flowcontrol/manager.go +++ b/les/flowcontrol/manager.go @@ -58,10 +58,9 @@ var ( // corrigated buffer value and usually allows a higher remaining buffer value // to be returned with each reply. type ClientManager struct { - clock mclock.Clock - lock sync.Mutex - enabledCh chan struct{} - stop chan chan struct{} + clock mclock.Clock + lock sync.Mutex + stop chan chan struct{} curve PieceWiseLinear sumRecharge, totalRecharge, totalConnected uint64 diff --git a/les/handler_test.go b/les/handler_test.go index aba45764b3068..56d7d55b5a5f7 100644 --- a/les/handler_test.go +++ b/les/handler_test.go @@ -405,7 +405,7 @@ func testGetProofs(t *testing.T, protocol int) { accounts := []common.Address{bankAddr, userAddr1, userAddr2, signerAddr, {}} for i := uint64(0); i <= bc.CurrentBlock().NumberU64(); i++ { header := bc.GetHeaderByNumber(i) - trie, _ := trie.New(header.Root, trie.NewDatabase(server.db)) + trie, _ := trie.New(common.Hash{}, header.Root, trie.NewDatabase(server.db)) for _, acc := range accounts { req := ProofReq{ @@ -456,7 +456,7 @@ func testGetStaleProof(t *testing.T, protocol int) { var expected []rlp.RawValue if wantOK { proofsV2 := light.NewNodeSet() - t, _ := trie.New(header.Root, trie.NewDatabase(server.db)) + t, _ := trie.New(common.Hash{}, header.Root, trie.NewDatabase(server.db)) t.Prove(account, 0, proofsV2) expected = proofsV2.NodeList() } @@ -512,7 +512,7 @@ func testGetCHTProofs(t *testing.T, protocol int) { AuxData: [][]byte{rlp}, } root := light.GetChtRoot(server.db, 0, bc.GetHeaderByNumber(config.ChtSize-1).Hash()) - trie, _ := trie.New(root, trie.NewDatabase(rawdb.NewTable(server.db, light.ChtTablePrefix))) + trie, _ := trie.New(common.Hash{}, root, trie.NewDatabase(rawdb.NewTable(server.db, light.ChtTablePrefix))) trie.Prove(key, 0, &proofsV2.Proofs) // Assemble the requests for the different protocols requestsV2 := []HelperTrieReq{{ @@ -577,7 +577,7 @@ func testGetBloombitsProofs(t *testing.T, protocol int) { var proofs HelperTrieResps root := light.GetBloomTrieRoot(server.db, 0, bc.GetHeaderByNumber(config.BloomTrieSize-1).Hash()) - trie, _ := trie.New(root, trie.NewDatabase(rawdb.NewTable(server.db, light.BloomTrieTablePrefix))) + trie, _ := trie.New(common.Hash{}, root, trie.NewDatabase(rawdb.NewTable(server.db, light.BloomTrieTablePrefix))) trie.Prove(key, 0, &proofs.Proofs) // Send the proof request and verify the response diff --git a/les/odr_test.go b/les/odr_test.go index ad77abf5b9b28..24b8e2ae31111 100644 --- a/les/odr_test.go +++ b/les/odr_test.go @@ -392,12 +392,10 @@ func testGetTxStatusFromUnindexedPeers(t *testing.T, protocol int) { for _, testspec := range testspecs { // Create a bunch of server peers with different tx history var ( - serverPeers []*testPeer - closeFns []func() + closeFns []func() ) for i := 0; i < testspec.peers; i++ { peer, closePeer, _ := client.newRawPeer(t, fmt.Sprintf("server-%d", i), protocol, testspec.txLookups[i]) - serverPeers = append(serverPeers, peer) closeFns = append(closeFns, closePeer) // Create a one-time routine for serving message diff --git a/les/peer.go b/les/peer.go index 499429739d23d..deda052a3b14b 100644 --- a/les/peer.go +++ b/les/peer.go @@ -995,40 +995,6 @@ func (p *clientPeer) sendLastAnnounce() { } } -// freezeClient temporarily puts the client in a frozen state which means all -// unprocessed and subsequent requests are dropped. Unfreezing happens automatically -// after a short time if the client's buffer value is at least in the slightly positive -// region. The client is also notified about being frozen/unfrozen with a Stop/Resume -// message. -func (p *clientPeer) freezeClient() { - if p.version < lpv3 { - // if Stop/Resume is not supported then just drop the peer after setting - // its frozen status permanently - atomic.StoreUint32(&p.frozen, 1) - p.Peer.Disconnect(p2p.DiscUselessPeer) - return - } - if atomic.SwapUint32(&p.frozen, 1) == 0 { - go func() { - p.sendStop() - time.Sleep(freezeTimeBase + time.Duration(rand.Int63n(int64(freezeTimeRandom)))) - for { - bufValue, bufLimit := p.fcClient.BufferStatus() - if bufLimit == 0 { - return - } - if bufValue <= bufLimit/8 { - time.Sleep(freezeCheckPeriod) - } else { - atomic.StoreUint32(&p.frozen, 0) - p.sendResume(bufValue) - break - } - } - }() - } -} - // Handshake executes the les protocol handshake, negotiating version number, // network IDs, difficulties, head and genesis blocks. func (p *clientPeer) Handshake(td *big.Int, head common.Hash, headNum uint64, genesis common.Hash, forkID forkid.ID, forkFilter forkid.Filter, server *LesServer) error { @@ -1157,19 +1123,6 @@ func (ps *serverPeerSet) subscribe(sub serverPeerSubscriber) { } } -// unSubscribe removes the specified service from the subscriber pool. -func (ps *serverPeerSet) unSubscribe(sub serverPeerSubscriber) { - ps.lock.Lock() - defer ps.lock.Unlock() - - for i, s := range ps.subscribers { - if s == sub { - ps.subscribers = append(ps.subscribers[:i], ps.subscribers[i+1:]...) - return - } - } -} - // register adds a new server peer into the set, or returns an error if the // peer is already known. func (ps *serverPeerSet) register(peer *serverPeer) error { @@ -1236,25 +1189,6 @@ func (ps *serverPeerSet) len() int { return len(ps.peers) } -// bestPeer retrieves the known peer with the currently highest total difficulty. -// If the peerset is "client peer set", then nothing meaningful will return. The -// reason is client peer never send back their latest status to server. -func (ps *serverPeerSet) bestPeer() *serverPeer { - ps.lock.RLock() - defer ps.lock.RUnlock() - - var ( - bestPeer *serverPeer - bestTd *big.Int - ) - for _, p := range ps.peers { - if td := p.Td(); bestTd == nil || td.Cmp(bestTd) > 0 { - bestPeer, bestTd = p, td - } - } - return bestPeer -} - // allServerPeers returns all server peers in a list. func (ps *serverPeerSet) allPeers() []*serverPeer { ps.lock.RLock() @@ -1348,14 +1282,6 @@ func (ps *clientPeerSet) peer(id enode.ID) *clientPeer { return ps.peers[id] } -// len returns if the current number of peers in the set. -func (ps *clientPeerSet) len() int { - ps.lock.RLock() - defer ps.lock.RUnlock() - - return len(ps.peers) -} - // setSignerKey sets the signer key for signed announcements. Should be called before // starting the protocol handler. func (ps *clientPeerSet) setSignerKey(privateKey *ecdsa.PrivateKey) { diff --git a/les/server_handler.go b/les/server_handler.go index ef1af844c26b1..a199a34a72545 100644 --- a/les/server_handler.go +++ b/les/server_handler.go @@ -359,7 +359,7 @@ func (h *serverHandler) AddTxsSync() bool { // getAccount retrieves an account from the state based on root. func getAccount(triedb *trie.Database, root, hash common.Hash) (types.StateAccount, error) { - trie, err := trie.New(root, triedb) + trie, err := trie.New(common.Hash{}, root, triedb) if err != nil { return types.StateAccount{}, err } @@ -391,7 +391,7 @@ func (h *serverHandler) GetHelperTrie(typ uint, index uint64) *trie.Trie { if root == (common.Hash{}) { return nil } - trie, _ := trie.New(root, trie.NewDatabase(rawdb.NewTable(h.chainDb, prefix))) + trie, _ := trie.New(common.Hash{}, root, trie.NewDatabase(rawdb.NewTable(h.chainDb, prefix))) return trie } diff --git a/les/ulc_test.go b/les/ulc_test.go index a4df0795b46d4..9a29a24cee55b 100644 --- a/les/ulc_test.go +++ b/les/ulc_test.go @@ -35,6 +35,19 @@ func TestULCAnnounceThresholdLes3(t *testing.T) { testULCAnnounceThreshold(t, 3) func testULCAnnounceThreshold(t *testing.T, protocol int) { // todo figure out why it takes fetcher so longer to fetcher the announced header. t.Skip("Sometimes it can failed") + + // newTestLightPeer creates node with light sync mode + newTestLightPeer := func(t *testing.T, protocol int, ulcServers []string, ulcFraction int) (*testClient, func()) { + netconfig := testnetConfig{ + protocol: protocol, + ulcServers: ulcServers, + ulcFraction: ulcFraction, + nopruning: true, + } + _, c, teardown := newClientServerEnv(t, netconfig) + return c, teardown + } + var cases = []struct { height []int threshold int @@ -148,15 +161,3 @@ func newTestServerPeer(t *testing.T, blocks int, protocol int, indexFn indexerCa n := enode.NewV4(&key.PublicKey, net.ParseIP("127.0.0.1"), 35000, 35000) return s, n, teardown } - -// newTestLightPeer creates node with light sync mode -func newTestLightPeer(t *testing.T, protocol int, ulcServers []string, ulcFraction int) (*testClient, func()) { - netconfig := testnetConfig{ - protocol: protocol, - ulcServers: ulcServers, - ulcFraction: ulcFraction, - nopruning: true, - } - _, c, teardown := newClientServerEnv(t, netconfig) - return c, teardown -} diff --git a/les/vflux/client/serverpool_test.go b/les/vflux/client/serverpool_test.go index c7d0245ef21d9..9f83c5f7f2c0f 100644 --- a/les/vflux/client/serverpool_test.go +++ b/les/vflux/client/serverpool_test.go @@ -55,7 +55,6 @@ type ServerPoolTest struct { clock *mclock.Simulated quit chan chan struct{} preNeg, preNegFail bool - vt *ValueTracker sp *ServerPool spi enode.Iterator input enode.Iterator diff --git a/les/vflux/server/clientpool.go b/les/vflux/server/clientpool.go index e90469bb1c9a4..734d74f453c93 100644 --- a/les/vflux/server/clientpool.go +++ b/les/vflux/server/clientpool.go @@ -61,7 +61,6 @@ type ClientPool struct { setup *serverSetup clock mclock.Clock - closed bool ns *nodestate.NodeStateMachine synced func() bool diff --git a/light/lightchain.go b/light/lightchain.go index fa0dc71c95996..2a8e3672145af 100644 --- a/light/lightchain.go +++ b/light/lightchain.go @@ -355,22 +355,6 @@ func (lc *LightChain) Rollback(chain []common.Hash) { } } -// postChainEvents iterates over the events generated by a chain insertion and -// posts them into the event feed. -func (lc *LightChain) postChainEvents(events []interface{}) { - for _, event := range events { - switch ev := event.(type) { - case core.ChainEvent: - if lc.CurrentHeader().Hash() == ev.Hash { - lc.chainHeadFeed.Send(core.ChainHeadEvent{Block: ev.Block}) - } - lc.chainFeed.Send(ev) - case core.ChainSideEvent: - lc.chainSideFeed.Send(ev) - } - } -} - func (lc *LightChain) InsertHeader(header *types.Header) error { // Verify the header first before obtaining the lock headers := []*types.Header{header} diff --git a/light/odr_test.go b/light/odr_test.go index fdf657a82ec5d..ec109f61f2bf5 100644 --- a/light/odr_test.go +++ b/light/odr_test.go @@ -82,7 +82,7 @@ func (odr *testOdr) Retrieve(ctx context.Context, req OdrRequest) error { req.Receipts = rawdb.ReadRawReceipts(odr.sdb, req.Hash, *number) } case *TrieRequest: - t, _ := trie.New(req.Id.Root, trie.NewDatabase(odr.sdb)) + t, _ := trie.New(common.BytesToHash(req.Id.AccKey), req.Id.Root, trie.NewDatabase(odr.sdb)) nodes := NewNodeSet() t.Prove(req.Key, 0, nodes) req.Proof = nodes diff --git a/light/postprocess.go b/light/postprocess.go index ce38d091e891a..c09b00e71c812 100644 --- a/light/postprocess.go +++ b/light/postprocess.go @@ -187,12 +187,12 @@ func (c *ChtIndexerBackend) Reset(ctx context.Context, section uint64, lastSecti root = GetChtRoot(c.diskdb, section-1, lastSectionHead) } var err error - c.trie, err = trie.New(root, c.triedb) + c.trie, err = trie.New(common.Hash{}, root, c.triedb) if err != nil && c.odr != nil { err = c.fetchMissingNodes(ctx, section, root) if err == nil { - c.trie, err = trie.New(root, c.triedb) + c.trie, err = trie.New(common.Hash{}, root, c.triedb) } } c.section = section @@ -253,9 +253,8 @@ func (c *ChtIndexerBackend) Commit() error { return nil } -// PruneSections implements core.ChainIndexerBackend which deletes all -// chain data(except hash<->number mappings) older than the specified -// threshold. +// Prune implements core.ChainIndexerBackend which deletes all chain data +// (except hash<->number mappings) older than the specified threshold. func (c *ChtIndexerBackend) Prune(threshold uint64) error { // Short circuit if the light pruning is disabled. if c.disablePruning { @@ -404,11 +403,11 @@ func (b *BloomTrieIndexerBackend) Reset(ctx context.Context, section uint64, las root = GetBloomTrieRoot(b.diskdb, section-1, lastSectionHead) } var err error - b.trie, err = trie.New(root, b.triedb) + b.trie, err = trie.New(common.Hash{}, root, b.triedb) if err != nil && b.odr != nil { err = b.fetchMissingNodes(ctx, section, root) if err == nil { - b.trie, err = trie.New(root, b.triedb) + b.trie, err = trie.New(common.Hash{}, root, b.triedb) } } b.section = section diff --git a/light/trie.go b/light/trie.go index 4ab6f4ace0756..931ba30cb40a2 100644 --- a/light/trie.go +++ b/light/trie.go @@ -169,7 +169,11 @@ func (t *odrTrie) do(key []byte, fn func() error) error { for { var err error if t.trie == nil { - t.trie, err = trie.New(t.id.Root, trie.NewDatabase(t.db.backend.Database())) + var owner common.Hash + if len(t.id.AccKey) > 0 { + owner = common.BytesToHash(t.id.AccKey) + } + t.trie, err = trie.New(owner, t.id.Root, trie.NewDatabase(t.db.backend.Database())) } if err == nil { err = fn() @@ -195,7 +199,11 @@ func newNodeIterator(t *odrTrie, startkey []byte) trie.NodeIterator { // Open the actual non-ODR trie if that hasn't happened yet. if t.trie == nil { it.do(func() error { - t, err := trie.New(t.id.Root, trie.NewDatabase(t.db.backend.Database())) + var owner common.Hash + if len(t.id.AccKey) > 0 { + owner = common.BytesToHash(t.id.AccKey) + } + t, err := trie.New(owner, t.id.Root, trie.NewDatabase(t.db.backend.Database())) if err == nil { it.t.trie = t } diff --git a/metrics/influxdb/influxdb.go b/metrics/influxdb/influxdb.go index dac9e824775a2..e99717aeebf9a 100644 --- a/metrics/influxdb/influxdb.go +++ b/metrics/influxdb/influxdb.go @@ -98,16 +98,16 @@ func (r *reporter) makeClient() (err error) { } func (r *reporter) run() { - intervalTicker := time.Tick(r.interval) - pingTicker := time.Tick(time.Second * 5) + intervalTicker := time.NewTicker(r.interval) + pingTicker := time.NewTicker(time.Second * 5) for { select { - case <-intervalTicker: + case <-intervalTicker.C: if err := r.send(); err != nil { log.Warn("Unable to send to InfluxDB", "err", err) } - case <-pingTicker: + case <-pingTicker.C: _, _, err := r.client.Ping() if err != nil { log.Warn("Got error while sending a ping to InfluxDB, trying to recreate client", "err", err) diff --git a/metrics/influxdb/influxdbv2.go b/metrics/influxdb/influxdbv2.go index 00901f52c9f4f..31f956481dae5 100644 --- a/metrics/influxdb/influxdbv2.go +++ b/metrics/influxdb/influxdbv2.go @@ -67,14 +67,14 @@ func InfluxDBV2WithTags(r metrics.Registry, d time.Duration, endpoint string, to } func (r *v2Reporter) run() { - intervalTicker := time.Tick(r.interval) - pingTicker := time.Tick(time.Second * 5) + intervalTicker := time.NewTicker(r.interval) + pingTicker := time.NewTicker(time.Second * 5) for { select { - case <-intervalTicker: + case <-intervalTicker.C: r.send() - case <-pingTicker: + case <-pingTicker.C: _, err := r.client.Health(context.Background()) if err != nil { log.Warn("Got error from influxdb client health check", "err", err.Error()) diff --git a/miner/stress/beacon/main.go b/miner/stress/beacon/main.go index 3f751049b89e6..439bcc5d10bd8 100644 --- a/miner/stress/beacon/main.go +++ b/miner/stress/beacon/main.go @@ -316,7 +316,7 @@ func (mgr *nodeManager) run() { nodes := mgr.getNodes(eth2MiningNode) nodes = append(nodes, mgr.getNodes(eth2NormalNode)...) nodes = append(nodes, mgr.getNodes(eth2LightClient)...) - for _, node := range append(nodes) { + for _, node := range nodes { fcState := beacon.ForkchoiceStateV1{ HeadBlockHash: oldest.Hash(), SafeBlockHash: common.Hash{}, diff --git a/miner/worker_test.go b/miner/worker_test.go index 55361349bcca6..bda0fd4899b01 100644 --- a/miner/worker_test.go +++ b/miner/worker_test.go @@ -113,7 +113,6 @@ type testWorkerBackend struct { db ethdb.Database txPool *core.TxPool chain *core.BlockChain - testTxFeed event.Feed genesis *core.Genesis uncleBlock *types.Block } diff --git a/node/config_test.go b/node/config_test.go index d9f812ec4cb2c..62b2fbbd16a3c 100644 --- a/node/config_test.go +++ b/node/config_test.go @@ -63,12 +63,9 @@ func TestDatadirCreation(t *testing.T) { }() dir = filepath.Join(file.Name(), "invalid/path") - node, err = New(&Config{DataDir: dir}) + _, err = New(&Config{DataDir: dir}) if err == nil { t.Fatalf("protocol stack created with an invalid datadir") - if err := node.Close(); err != nil { - t.Fatalf("failed to close node: %v", err) - } } } diff --git a/node/node.go b/node/node.go index 7c540306db2b1..0a2b9eb836920 100644 --- a/node/node.go +++ b/node/node.go @@ -20,6 +20,7 @@ import ( crand "crypto/rand" "errors" "fmt" + "hash/crc32" "net/http" "os" "path/filepath" @@ -352,10 +353,10 @@ func (n *Node) obtainJWTSecret(cliParam string) ([]byte, error) { fileName = n.ResolvePath(datadirJWTKey) } // try reading from file - log.Debug("Reading JWT secret", "path", fileName) if data, err := os.ReadFile(fileName); err == nil { jwtSecret := common.FromHex(strings.TrimSpace(string(data))) if len(jwtSecret) == 32 { + log.Info("Loaded JWT secret file", "path", fileName, "crc32", fmt.Sprintf("%#x", crc32.ChecksumIEEE(jwtSecret))) return jwtSecret, nil } log.Error("Invalid JWT secret", "path", fileName, "length", len(jwtSecret)) diff --git a/node/rpcstack_test.go b/node/rpcstack_test.go index c7dba8a1ef0d1..229a5b5e53baa 100644 --- a/node/rpcstack_test.go +++ b/node/rpcstack_test.go @@ -314,6 +314,11 @@ func TestJWT(t *testing.T) { ss, _ := jwt.NewWithClaims(method, testClaim(input)).SignedString(secret) return ss } + srv := createAndStartServer(t, &httpConfig{jwtSecret: []byte("secret")}, + true, &wsConfig{Origins: []string{"*"}, jwtSecret: []byte("secret")}) + wsUrl := fmt.Sprintf("ws://%v", srv.listenAddr()) + htUrl := fmt.Sprintf("http://%v", srv.listenAddr()) + expOk := []string{ fmt.Sprintf("Bearer %v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix()})), fmt.Sprintf("Bearer %v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix() + 4})), @@ -327,6 +332,14 @@ func TestJWT(t *testing.T) { "bar": "baz", })), } + for i, token := range expOk { + if err := wsRequest(t, wsUrl, "Authorization", token); err != nil { + t.Errorf("test %d-ws, token '%v': expected ok, got %v", i, token, err) + } + if resp := rpcRequest(t, htUrl, "Authorization", token); resp.StatusCode != 200 { + t.Errorf("test %d-http, token '%v': expected ok, got %v", i, token, resp.StatusCode) + } + } expFail := []string{ // future fmt.Sprintf("Bearer %v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix() + 6})), @@ -351,19 +364,6 @@ func TestJWT(t *testing.T) { fmt.Sprintf("Bearer\t%v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix()})), fmt.Sprintf("Bearer \t%v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix()})), } - srv := createAndStartServer(t, &httpConfig{jwtSecret: []byte("secret")}, - true, &wsConfig{Origins: []string{"*"}, jwtSecret: []byte("secret")}) - wsUrl := fmt.Sprintf("ws://%v", srv.listenAddr()) - htUrl := fmt.Sprintf("http://%v", srv.listenAddr()) - - for i, token := range expOk { - if err := wsRequest(t, wsUrl, "Authorization", token); err != nil { - t.Errorf("test %d-ws, token '%v': expected ok, got %v", i, token, err) - } - if resp := rpcRequest(t, htUrl, "Authorization", token); resp.StatusCode != 200 { - t.Errorf("test %d-http, token '%v': expected ok, got %v", i, token, resp.StatusCode) - } - } for i, token := range expFail { if err := wsRequest(t, wsUrl, "Authorization", token); err == nil { t.Errorf("tc %d-ws, token '%v': expected not to allow, got ok", i, token) diff --git a/node/utils_test.go b/node/utils_test.go index b7474bb706180..1974e2e17cdd1 100644 --- a/node/utils_test.go +++ b/node/utils_test.go @@ -47,8 +47,6 @@ type InstrumentedService struct { startHook func() stopHook func() - - protocols []p2p.Protocol } func (s *InstrumentedService) Start() error { diff --git a/p2p/discover/lookup.go b/p2p/discover/lookup.go index 9ab4a71ce7b42..b8d97b44e1cc5 100644 --- a/p2p/discover/lookup.go +++ b/p2p/discover/lookup.go @@ -18,6 +18,7 @@ package discover import ( "context" + "errors" "time" "github.com/ethereum/go-ethereum/p2p/enode" @@ -141,7 +142,7 @@ func (it *lookup) slowdown() { func (it *lookup) query(n *node, reply chan<- []*node) { fails := it.tab.db.FindFails(n.ID(), n.IP()) r, err := it.queryfunc(n) - if err == errClosed { + if errors.Is(err, errClosed) { // Avoid recording failures on shutdown. reply <- nil return diff --git a/p2p/discover/v4_udp.go b/p2p/discover/v4_udp.go index 334716aebed01..a3e1549075b36 100644 --- a/p2p/discover/v4_udp.go +++ b/p2p/discover/v4_udp.go @@ -328,7 +328,7 @@ func (t *UDPv4) findnode(toid enode.ID, toaddr *net.UDPAddr, target v4wire.Pubke // enough nodes the reply matcher will time out waiting for the second reply, but // there's no need for an error in that case. err := <-rm.errc - if err == errTimeout && rm.reply != nil { + if errors.Is(err, errTimeout) && rm.reply != nil { err = nil } return nodes, err @@ -526,7 +526,7 @@ func (t *UDPv4) readLoop(unhandled chan<- ReadPacket) { continue } else if err != nil { // Shut down the loop for permament errors. - if err != io.EOF { + if !errors.Is(err, io.EOF) { t.log.Debug("UDP read error", "err", err) } return diff --git a/p2p/discover/v4_udp_test.go b/p2p/discover/v4_udp_test.go index e36912f010aeb..977fa85a06c6c 100644 --- a/p2p/discover/v4_udp_test.go +++ b/p2p/discover/v4_udp_test.go @@ -284,6 +284,7 @@ func TestUDPv4_findnode(t *testing.T) { test.waitPacketOut(func(p *v4wire.Neighbors, to *net.UDPAddr, hash []byte) { if len(p.Nodes) != len(want) { t.Errorf("wrong number of results: got %d, want %d", len(p.Nodes), bucketSize) + return } for i, n := range p.Nodes { if n.ID.ID() != want[i].ID() { diff --git a/p2p/discover/v5_udp.go b/p2p/discover/v5_udp.go index dc63382fc9011..22fab72435019 100644 --- a/p2p/discover/v5_udp.go +++ b/p2p/discover/v5_udp.go @@ -305,7 +305,7 @@ func (t *UDPv5) lookupWorker(destNode *node, target enode.ID) ([]*node, error) { ) var r []*enode.Node r, err = t.findnode(unwrapNode(destNode), dists) - if err == errClosed { + if errors.Is(err, errClosed) { return nil, err } for _, n := range r { @@ -623,7 +623,7 @@ func (t *UDPv5) readLoop() { continue } else if err != nil { // Shut down the loop for permament errors. - if err != io.EOF { + if !errors.Is(err, io.EOF) { t.log.Debug("UDP read error", "err", err) } return diff --git a/p2p/discover/v5wire/encoding.go b/p2p/discover/v5wire/encoding.go index 7d17281ef9691..8dc64de6dfd68 100644 --- a/p2p/discover/v5wire/encoding.go +++ b/p2p/discover/v5wire/encoding.go @@ -596,7 +596,7 @@ func (c *Codec) decodeMessage(fromAddr string, head *Header, headerData, msgData // Try decrypting the message. key := c.sc.readKey(auth.SrcID, fromAddr) msg, err := c.decryptMessage(msgData, head.Nonce[:], headerData, key) - if err == errMessageDecrypt { + if errors.Is(err, errMessageDecrypt) { // It didn't work. Start the handshake since this is an ordinary message packet. return &Unknown{Nonce: head.Nonce}, nil } diff --git a/p2p/dnsdisc/client.go b/p2p/dnsdisc/client.go index 93868b39a8d4b..3f914d6e94164 100644 --- a/p2p/dnsdisc/client.go +++ b/p2p/dnsdisc/client.go @@ -19,6 +19,7 @@ package dnsdisc import ( "bytes" "context" + "errors" "fmt" "math/rand" "net" @@ -204,7 +205,7 @@ func (c *Client) doResolveEntry(ctx context.Context, domain, hash string) (entry } for _, txt := range txts { e, err := parseEntry(txt, c.cfg.ValidSchemes) - if err == errUnknownEntry { + if errors.Is(err, errUnknownEntry) { continue } if !bytes.HasPrefix(crypto.Keccak256([]byte(txt)), wantHash) { @@ -281,7 +282,7 @@ func (it *randomIterator) nextNode() *enode.Node { } n, err := ct.syncRandom(it.ctx) if err != nil { - if err == it.ctx.Err() { + if errors.Is(err, it.ctx.Err()) { return nil // context canceled. } it.c.cfg.Logger.Debug("Error in DNS random node sync", "tree", ct.loc.domain, "err", err) diff --git a/p2p/enr/entries.go b/p2p/enr/entries.go index f2118401afb85..a8b0a3839bda5 100644 --- a/p2p/enr/entries.go +++ b/p2p/enr/entries.go @@ -17,6 +17,7 @@ package enr import ( + "errors" "fmt" "io" "net" @@ -180,9 +181,16 @@ func (err *KeyError) Error() string { return fmt.Sprintf("ENR key %q: %v", err.Key, err.Err) } +func (err *KeyError) Unwrap() error { + return err.Err +} + // IsNotFound reports whether the given error means that a key/value pair is // missing from a record. func IsNotFound(err error) bool { - kerr, ok := err.(*KeyError) - return ok && kerr.Err == errNotFound + var ke *KeyError + if errors.As(err, &ke) { + return ke.Err == errNotFound + } + return false } diff --git a/p2p/nat/natupnp.go b/p2p/nat/natupnp.go index 1f5d714664504..a8de00e978b92 100644 --- a/p2p/nat/natupnp.go +++ b/p2p/nat/natupnp.go @@ -79,7 +79,7 @@ func (n *upnp) ExternalIP() (addr net.IP, err error) { func (n *upnp) AddMapping(protocol string, extport, intport int, desc string, lifetime time.Duration) error { ip, err := n.internalAddress() if err != nil { - return nil + return nil // TODO: Shouldn't we return the error? } protocol = strings.ToUpper(protocol) lifetimeS := uint32(lifetime / time.Second) diff --git a/p2p/peer.go b/p2p/peer.go index 257027a5b74d1..469a1b797416d 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -416,7 +416,7 @@ func (p *Peer) startProtocols(writeStart <-chan struct{}, writeErr chan<- error) if err == nil { p.log.Trace(fmt.Sprintf("Protocol %s/%d returned", proto.Name, proto.Version)) err = errProtocolReturned - } else if err != io.EOF { + } else if !errors.Is(err, io.EOF) { p.log.Trace(fmt.Sprintf("Protocol %s/%d failed", proto.Name, proto.Version), "err", err) } p.protoErr <- err diff --git a/p2p/peer_error.go b/p2p/peer_error.go index 3028685041fe3..ebc59de251a8b 100644 --- a/p2p/peer_error.go +++ b/p2p/peer_error.go @@ -103,7 +103,7 @@ func discReasonForError(err error) DiscReason { if reason, ok := err.(DiscReason); ok { return reason } - if err == errProtocolReturned { + if errors.Is(err, errProtocolReturned) { return DiscQuitting } peerError, ok := err.(*peerError) diff --git a/p2p/server.go b/p2p/server.go index 138975e54bf5f..898201f8f7fc2 100644 --- a/p2p/server.go +++ b/p2p/server.go @@ -126,7 +126,7 @@ type Config struct { // Protocols should contain the protocols supported // by the server. Matching protocols are launched for // each peer. - Protocols []Protocol `toml:"-"` + Protocols []Protocol `toml:"-" json:"-"` // If ListenAddr is set to a non-nil address, the server // will listen for incoming connections. diff --git a/p2p/simulations/http.go b/p2p/simulations/http.go index 341ff8718b7df..a344a8d5cbbe6 100644 --- a/p2p/simulations/http.go +++ b/p2p/simulations/http.go @@ -21,6 +21,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "html" "io" @@ -441,6 +442,7 @@ func (s *Server) StreamNetworkEvents(w http.ResponseWriter, req *http.Request) { } } for _, conn := range snap.Conns { + conn := conn event := NewEvent(&conn) if err := writeEvent(event); err != nil { writeErr(err) @@ -559,7 +561,7 @@ func (s *Server) CreateNode(w http.ResponseWriter, req *http.Request) { config := &adapters.NodeConfig{} err := json.NewDecoder(req.Body).Decode(config) - if err != nil && err != io.EOF { + if err != nil && !errors.Is(err, io.EOF) { http.Error(w, err.Error(), http.StatusBadRequest) return } diff --git a/params/config.go b/params/config.go index 3533bf1c7918a..44b575a876c70 100644 --- a/params/config.go +++ b/params/config.go @@ -73,6 +73,7 @@ var ( BerlinBlock: big.NewInt(12_244_000), LondonBlock: big.NewInt(12_965_000), ArrowGlacierBlock: big.NewInt(13_773_000), + GrayGlacierBlock: big.NewInt(15_050_000), Ethash: new(EthashConfig), } @@ -114,7 +115,7 @@ var ( MuirGlacierBlock: big.NewInt(7_117_117), BerlinBlock: big.NewInt(9_812_189), LondonBlock: big.NewInt(10_499_401), - TerminalTotalDifficulty: big.NewInt(43531756765713534), + TerminalTotalDifficulty: new(big.Int).SetUint64(50000000000000000), Ethash: new(EthashConfig), } @@ -259,19 +260,28 @@ var ( // // This configuration is intentionally not using keyed fields to force anyone // adding flags to the config to also have to set these fields. - AllEthashProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, new(EthashConfig), nil} + AllEthashProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, new(EthashConfig), nil} // AllCliqueProtocolChanges contains every protocol change (EIPs) introduced // and accepted by the Ethereum core developers into the Clique consensus. // // This configuration is intentionally not using keyed fields to force anyone // adding flags to the config to also have to set these fields. - AllCliqueProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, nil, nil, &CliqueConfig{Period: 0, Epoch: 30000}} + AllCliqueProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, nil, nil, nil, &CliqueConfig{Period: 0, Epoch: 30000}} - TestChainConfig = &ChainConfig{big.NewInt(1), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, new(EthashConfig), nil} + TestChainConfig = &ChainConfig{big.NewInt(1), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, new(EthashConfig), nil} TestRules = TestChainConfig.Rules(new(big.Int), false) ) +// NetworkNames are user friendly names to use in the chain spec banner. +var NetworkNames = map[string]string{ + MainnetChainConfig.ChainID.String(): "mainnet", + RopstenChainConfig.ChainID.String(): "ropsten", + RinkebyChainConfig.ChainID.String(): "rinkeby", + GoerliChainConfig.ChainID.String(): "goerli", + SepoliaChainConfig.ChainID.String(): "sepolia", +} + // TrustedCheckpoint represents a set of post-processed trie roots (CHT and // BloomTrie) associated with the appropriate section index and head hash. It is // used to start light syncing from this checkpoint and avoid downloading the @@ -348,7 +358,8 @@ type ChainConfig struct { BerlinBlock *big.Int `json:"berlinBlock,omitempty"` // Berlin switch block (nil = no fork, 0 = already on berlin) LondonBlock *big.Int `json:"londonBlock,omitempty"` // London switch block (nil = no fork, 0 = already on london) ArrowGlacierBlock *big.Int `json:"arrowGlacierBlock,omitempty"` // Eip-4345 (bomb delay) switch block (nil = no fork, 0 = already activated) - MergeForkBlock *big.Int `json:"mergeForkBlock,omitempty"` // EIP-3675 (TheMerge) switch block (nil = no fork, 0 = already in merge proceedings) + GrayGlacierBlock *big.Int `json:"grayGlacierBlock,omitempty"` // Eip-5133 (bomb delay) switch block (nil = no fork, 0 = already activated) + MergeNetsplitBlock *big.Int `json:"mergeNetsplitBlock,omitempty"` // Virtual fork after The Merge to use as a network splitter // TerminalTotalDifficulty is the amount of total difficulty reached by // the network that triggers the consensus upgrade. @@ -380,35 +391,71 @@ func (c *CliqueConfig) String() string { // String implements the fmt.Stringer interface. func (c *ChainConfig) String() string { - var engine interface{} + var banner string + + // Create some basinc network config output + network := NetworkNames[c.ChainID.String()] + if network == "" { + network = "unknown" + } + banner += fmt.Sprintf("Chain ID: %v (%s)\n", c.ChainID, network) switch { case c.Ethash != nil: - engine = c.Ethash + if c.TerminalTotalDifficulty == nil { + banner += "Consensus: Ethash (proof-of-work)\n" + } else { + banner += "Consensus: Beacon (proof-of-stake), merged from Ethash (proof-of-work)\n" + } case c.Clique != nil: - engine = c.Clique + if c.TerminalTotalDifficulty == nil { + banner += "Consensus: Clique (proof-of-authority)\n" + } else { + banner += "Consensus: Beacon (proof-of-stake), merged from Clique (proof-of-authority)\n" + } default: - engine = "unknown" - } - return fmt.Sprintf("{ChainID: %v Homestead: %v DAO: %v DAOSupport: %v EIP150: %v EIP155: %v EIP158: %v Byzantium: %v Constantinople: %v Petersburg: %v Istanbul: %v, Muir Glacier: %v, Berlin: %v, London: %v, Arrow Glacier: %v, MergeFork: %v, Terminal TD: %v, Engine: %v}", - c.ChainID, - c.HomesteadBlock, - c.DAOForkBlock, - c.DAOForkSupport, - c.EIP150Block, - c.EIP155Block, - c.EIP158Block, - c.ByzantiumBlock, - c.ConstantinopleBlock, - c.PetersburgBlock, - c.IstanbulBlock, - c.MuirGlacierBlock, - c.BerlinBlock, - c.LondonBlock, - c.ArrowGlacierBlock, - c.MergeForkBlock, - c.TerminalTotalDifficulty, - engine, - ) + banner += "Consensus: unknown\n" + } + banner += "\n" + + // Create a list of forks with a short description of them. Forks that only + // makes sense for mainnet should be optional at printing to avoid bloating + // the output for testnets and private networks. + banner += "Pre-Merge hard forks:\n" + banner += fmt.Sprintf(" - Homestead: %-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/homestead.md)\n", c.HomesteadBlock) + if c.DAOForkBlock != nil { + banner += fmt.Sprintf(" - DAO Fork: %-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/dao-fork.md)\n", c.DAOForkBlock) + } + banner += fmt.Sprintf(" - Tangerine Whistle (EIP 150): %-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/tangerine-whistle.md)\n", c.EIP150Block) + banner += fmt.Sprintf(" - Spurious Dragon/1 (EIP 155): %-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/spurious-dragon.md)\n", c.EIP155Block) + banner += fmt.Sprintf(" - Spurious Dragon/2 (EIP 158): %-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/spurious-dragon.md)\n", c.EIP155Block) + banner += fmt.Sprintf(" - Byzantium: %-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/byzantium.md)\n", c.ByzantiumBlock) + banner += fmt.Sprintf(" - Constantinople: %-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/constantinople.md)\n", c.ConstantinopleBlock) + banner += fmt.Sprintf(" - Petersburg: %-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/petersburg.md)\n", c.PetersburgBlock) + banner += fmt.Sprintf(" - Istanbul: %-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/istanbul.md)\n", c.IstanbulBlock) + if c.MuirGlacierBlock != nil { + banner += fmt.Sprintf(" - Muir Glacier: %-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/muir-glacier.md)\n", c.MuirGlacierBlock) + } + banner += fmt.Sprintf(" - Berlin: %-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/berlin.md)\n", c.BerlinBlock) + banner += fmt.Sprintf(" - London: %-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/london.md)\n", c.LondonBlock) + if c.ArrowGlacierBlock != nil { + banner += fmt.Sprintf(" - Arrow Glacier: %-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/arrow-glacier.md)\n", c.ArrowGlacierBlock) + } + if c.GrayGlacierBlock != nil { + banner += fmt.Sprintf(" - Gray Glacier: %-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/gray-glacier.md)\n", c.GrayGlacierBlock) + } + banner += "\n" + + // Add a special section for the merge as it's non-obvious + if c.TerminalTotalDifficulty == nil { + banner += "Merge not configured!\n" + banner += " - Hard-fork specification: https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/paris.md)" + } else { + banner += "Merge configured:\n" + banner += " - Hard-fork specification: https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/paris.md)\n" + banner += fmt.Sprintf(" - Total terminal difficulty: %v\n", c.TerminalTotalDifficulty) + banner += fmt.Sprintf(" - Merge netsplit block: %-8v", c.MergeNetsplitBlock) + } + return banner } // IsHomestead returns whether num is either equal to the homestead block or greater. @@ -478,6 +525,11 @@ func (c *ChainConfig) IsArrowGlacier(num *big.Int) bool { return isForked(c.ArrowGlacierBlock, num) } +// IsGrayGlacier returns whether num is either equal to the Gray Glacier (EIP-5133) fork block or greater. +func (c *ChainConfig) IsGrayGlacier(num *big.Int) bool { + return isForked(c.GrayGlacierBlock, num) +} + // IsTerminalPoWBlock returns whether the given block is the last block of PoW stage. func (c *ChainConfig) IsTerminalPoWBlock(parentTotalDiff *big.Int, totalDiff *big.Int) bool { if c.TerminalTotalDifficulty == nil { @@ -527,7 +579,8 @@ func (c *ChainConfig) CheckConfigForkOrder() error { {name: "berlinBlock", block: c.BerlinBlock}, {name: "londonBlock", block: c.LondonBlock}, {name: "arrowGlacierBlock", block: c.ArrowGlacierBlock, optional: true}, - {name: "mergeStartBlock", block: c.MergeForkBlock, optional: true}, + {name: "grayGlacierBlock", block: c.GrayGlacierBlock, optional: true}, + {name: "mergeNetsplitBlock", block: c.MergeNetsplitBlock, optional: true}, } { if lastFork.name != "" { // Next one must be higher number @@ -600,8 +653,11 @@ func (c *ChainConfig) checkCompatible(newcfg *ChainConfig, head *big.Int) *Confi if isForkIncompatible(c.ArrowGlacierBlock, newcfg.ArrowGlacierBlock, head) { return newCompatError("Arrow Glacier fork block", c.ArrowGlacierBlock, newcfg.ArrowGlacierBlock) } - if isForkIncompatible(c.MergeForkBlock, newcfg.MergeForkBlock, head) { - return newCompatError("Merge Start fork block", c.MergeForkBlock, newcfg.MergeForkBlock) + if isForkIncompatible(c.GrayGlacierBlock, newcfg.GrayGlacierBlock, head) { + return newCompatError("Gray Glacier fork block", c.GrayGlacierBlock, newcfg.GrayGlacierBlock) + } + if isForkIncompatible(c.MergeNetsplitBlock, newcfg.MergeNetsplitBlock, head) { + return newCompatError("Merge netsplit fork block", c.MergeNetsplitBlock, newcfg.MergeNetsplitBlock) } return nil } diff --git a/rlp/decode_test.go b/rlp/decode_test.go index 46aa68cea3d73..e0d33dc43ee91 100644 --- a/rlp/decode_test.go +++ b/rlp/decode_test.go @@ -452,7 +452,7 @@ type ignoredField struct { var ( veryBigInt = new(big.Int).Add( - big.NewInt(0).Lsh(big.NewInt(0xFFFFFFFFFFFFFF), 16), + new(big.Int).Lsh(big.NewInt(0xFFFFFFFFFFFFFF), 16), big.NewInt(0xFFFF), ) veryVeryBigInt = new(big.Int).Exp(veryBigInt, big.NewInt(8), nil) diff --git a/rlp/encode_test.go b/rlp/encode_test.go index 78392906b557e..58ddc0d120f0f 100644 --- a/rlp/encode_test.go +++ b/rlp/encode_test.go @@ -119,15 +119,15 @@ var encTests = []encTest{ {val: big.NewInt(0xFFFFFFFFFFFF), output: "86FFFFFFFFFFFF"}, {val: big.NewInt(0xFFFFFFFFFFFFFF), output: "87FFFFFFFFFFFFFF"}, { - val: big.NewInt(0).SetBytes(unhex("102030405060708090A0B0C0D0E0F2")), + val: new(big.Int).SetBytes(unhex("102030405060708090A0B0C0D0E0F2")), output: "8F102030405060708090A0B0C0D0E0F2", }, { - val: big.NewInt(0).SetBytes(unhex("0100020003000400050006000700080009000A000B000C000D000E01")), + val: new(big.Int).SetBytes(unhex("0100020003000400050006000700080009000A000B000C000D000E01")), output: "9C0100020003000400050006000700080009000A000B000C000D000E01", }, { - val: big.NewInt(0).SetBytes(unhex("010000000000000000000000000000000000000000000000000000000000000000")), + val: new(big.Int).SetBytes(unhex("010000000000000000000000000000000000000000000000000000000000000000")), output: "A1010000000000000000000000000000000000000000000000000000000000000000", }, { diff --git a/rlp/rlpgen/main.go b/rlp/rlpgen/main.go index 6258fdb47a563..17d7e64e0842b 100644 --- a/rlp/rlpgen/main.go +++ b/rlp/rlpgen/main.go @@ -51,7 +51,7 @@ func main() { } if *output == "-" { os.Stdout.Write(code) - } else if err := os.WriteFile(*output, code, 0644); err != nil { + } else if err := os.WriteFile(*output, code, 0600); err != nil { fatal(err) } } diff --git a/rpc/client_test.go b/rpc/client_test.go index fa6010bb199c1..04c847d0d626c 100644 --- a/rpc/client_test.go +++ b/rpc/client_test.go @@ -615,10 +615,10 @@ func TestClientReconnect(t *testing.T) { // Start a server and corresponding client. s1, l1 := startServer("127.0.0.1:0") client, err := DialContext(ctx, "ws://"+l1.Addr().String()) - defer client.Close() if err != nil { t.Fatal("can't dial", err) } + defer client.Close() // Perform a call. This should work because the server is up. var resp echoResult diff --git a/rpc/handler.go b/rpc/handler.go index e8d1887c7d236..cd95a067f3e26 100644 --- a/rpc/handler.go +++ b/rpc/handler.go @@ -346,7 +346,7 @@ func (h *handler) handleCall(cp *callProc, msg *jsonrpcMessage) *jsonrpcMessage successfulRequestGauge.Inc(1) } rpcServingTimer.UpdateSince(start) - newRPCServingTimer(msg.Method, answer.Error == nil).UpdateSince(start) + updateServeTimeHistogram(msg.Method, answer.Error == nil, time.Since(start)) } return answer } diff --git a/rpc/metrics.go b/rpc/metrics.go index 4f166ad1cc073..b1f1284535e26 100644 --- a/rpc/metrics.go +++ b/rpc/metrics.go @@ -18,6 +18,7 @@ package rpc import ( "fmt" + "time" "github.com/ethereum/go-ethereum/metrics" ) @@ -26,14 +27,24 @@ var ( rpcRequestGauge = metrics.NewRegisteredGauge("rpc/requests", nil) successfulRequestGauge = metrics.NewRegisteredGauge("rpc/success", nil) failedRequestGauge = metrics.NewRegisteredGauge("rpc/failure", nil) - rpcServingTimer = metrics.NewRegisteredTimer("rpc/duration/all", nil) + + // serveTimeHistName is the prefix of the per-request serving time histograms. + serveTimeHistName = "rpc/duration" + + rpcServingTimer = metrics.NewRegisteredTimer("rpc/duration/all", nil) ) -func newRPCServingTimer(method string, valid bool) metrics.Timer { - flag := "success" - if !valid { - flag = "failure" +// updateServeTimeHistogram tracks the serving time of a remote RPC call. +func updateServeTimeHistogram(method string, success bool, elapsed time.Duration) { + note := "success" + if !success { + note = "failure" + } + h := fmt.Sprintf("%s/%s/%s", serveTimeHistName, method, note) + sampler := func() metrics.Sample { + return metrics.ResettingSample( + metrics.NewExpDecaySample(1028, 0.015), + ) } - m := fmt.Sprintf("rpc/duration/%s/%s", method, flag) - return metrics.GetOrRegisterTimer(m, nil) + metrics.GetOrRegisterHistogramLazy(h, nil, sampler).Update(elapsed.Microseconds()) } diff --git a/signer/core/api_test.go b/signer/core/api_test.go index ddc2b82eac69b..821af663fa81a 100644 --- a/signer/core/api_test.go +++ b/signer/core/api_test.go @@ -62,7 +62,7 @@ func (ui *headlessUi) ApproveTx(request *core.SignTxRequest) (core.SignTxRespons case "M": // modify // The headless UI always modifies the transaction old := big.Int(request.Transaction.Value) - newVal := big.NewInt(0).Add(&old, big.NewInt(1)) + newVal := new(big.Int).Add(&old, big.NewInt(1)) request.Transaction.Value = hexutil.Big(*newVal) return core.SignTxResponse{request.Transaction, true}, nil default: diff --git a/signer/core/signed_data.go b/signer/core/signed_data.go index 9bf47be799d8d..48559bd98cf72 100644 --- a/signer/core/signed_data.go +++ b/signer/core/signed_data.go @@ -169,7 +169,7 @@ func (api *SignerAPI) determineSignatureFormat(ctx context.Context, contentType req = &SignDataRequest{ContentType: mediaType, Rawdata: cliqueRlp, Messages: messages, Hash: sighash} default: // also case TextPlain.Mime: // Calculates an Ethereum ECDSA signature for: - // hash = keccak256("\x19${byteVersion}Ethereum Signed Message:\n${message length}${message}") + // hash = keccak256("\x19Ethereum Signed Message:\n${message length}${message}") // We expect it to be a string if stringData, ok := data.(string); !ok { return nil, useEthereumV, fmt.Errorf("input for text/plain must be an hex-encoded string") @@ -194,7 +194,7 @@ func (api *SignerAPI) determineSignatureFormat(ctx context.Context, contentType return req, useEthereumV, nil } -// SignTextWithValidator signs the given message which can be further recovered +// SignTextValidator signs the given message which can be further recovered // with the given validator. // hash = keccak256("\x19\x00"${address}${data}). func SignTextValidator(validatorData apitypes.ValidatorData) (hexutil.Bytes, string) { @@ -271,11 +271,11 @@ func (api *SignerAPI) EcRecover(ctx context.Context, data hexutil.Bytes, sig hex // // Note, this function is compatible with eth_sign and personal_sign. As such it recovers // the address of: - // hash = keccak256("\x19${byteVersion}Ethereum Signed Message:\n${message length}${message}") + // hash = keccak256("\x19Ethereum Signed Message:\n${message length}${message}") // addr = ecrecover(hash, signature) // // Note, the signature must conform to the secp256k1 curve R, S and V values, where - // the V value must be be 27 or 28 for legacy reasons. + // the V value must be 27 or 28 for legacy reasons. // // https://github.com/ethereum/go-ethereum/wiki/Management-APIs#personal_ecRecover if len(sig) != 65 { diff --git a/signer/fourbyte/validation_test.go b/signer/fourbyte/validation_test.go index c3085696f4326..2e6d9f2d9bb77 100644 --- a/signer/fourbyte/validation_test.go +++ b/signer/fourbyte/validation_test.go @@ -29,11 +29,11 @@ func mixAddr(a string) (*common.MixedcaseAddress, error) { return common.NewMixedcaseAddressFromString(a) } func toHexBig(h string) hexutil.Big { - b := big.NewInt(0).SetBytes(common.FromHex(h)) + b := new(big.Int).SetBytes(common.FromHex(h)) return hexutil.Big(*b) } func toHexUint(h string) hexutil.Uint64 { - b := big.NewInt(0).SetBytes(common.FromHex(h)) + b := new(big.Int).SetBytes(common.FromHex(h)) return hexutil.Uint64(b.Uint64()) } func dummyTxArgs(t txtestcase) *apitypes.SendTxArgs { diff --git a/signer/rules/rules_test.go b/signer/rules/rules_test.go index 0ab246eeaf7e7..af4ed2d7e1321 100644 --- a/signer/rules/rules_test.go +++ b/signer/rules/rules_test.go @@ -449,7 +449,7 @@ func dummyTx(value hexutil.Big) *core.SignTxRequest { } func dummyTxWithV(value uint64) *core.SignTxRequest { - v := big.NewInt(0).SetUint64(value) + v := new(big.Int).SetUint64(value) h := hexutil.Big(*v) return dummyTx(h) } @@ -469,7 +469,7 @@ func TestLimitWindow(t *testing.T) { return } // 0.3 ether: 429D069189E0000 wei - v := big.NewInt(0).SetBytes(common.Hex2Bytes("0429D069189E0000")) + v := new(big.Int).SetBytes(common.Hex2Bytes("0429D069189E0000")) h := hexutil.Big(*v) // The first three should succeed for i := 0; i < 3; i++ { diff --git a/tests/difficulty_test.go b/tests/difficulty_test.go index 192dff12cc97c..8239850b76177 100644 --- a/tests/difficulty_test.go +++ b/tests/difficulty_test.go @@ -79,6 +79,9 @@ func TestDifficulty(t *testing.T) { dt.config("EIP4345", params.ChainConfig{ ArrowGlacierBlock: big.NewInt(0), }) + dt.config("EIP5133", params.ChainConfig{ + GrayGlacierBlock: big.NewInt(0), + }) dt.config("difficulty.json", mainnetChainConfig) dt.walk(t, difficultyTestDir, func(t *testing.T, name string, test *DifficultyTest) { diff --git a/tests/fuzzers/bls12381/bls12381_fuzz.go b/tests/fuzzers/bls12381/bls12381_fuzz.go index b283ed11fe395..a5b4b9e798569 100644 --- a/tests/fuzzers/bls12381/bls12381_fuzz.go +++ b/tests/fuzzers/bls12381/bls12381_fuzz.go @@ -29,20 +29,22 @@ import ( gnark "github.com/consensys/gnark-crypto/ecc/bls12-381" "github.com/consensys/gnark-crypto/ecc/bls12-381/fp" "github.com/consensys/gnark-crypto/ecc/bls12-381/fr" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto/bls12381" + blst "github.com/supranational/blst/bindings/go" ) func FuzzCrossPairing(data []byte) int { input := bytes.NewReader(data) // get random G1 points - kpG1, cpG1, err := getG1Points(input) + kpG1, cpG1, blG1, err := getG1Points(input) if err != nil { return 0 } // get random G2 points - kpG2, cpG2, err := getG2Points(input) + kpG2, cpG2, blG2, err := getG2Points(input) if err != nil { return 0 } @@ -63,20 +65,52 @@ func FuzzCrossPairing(data []byte) int { panic("pairing mismatch gnark / geth ") } + // compute pairing using blst + blstResult := blst.Fp12MillerLoop(blG2, blG1) + blstResult.FinalExp() + res := massageBLST(blstResult.ToBendian()) + if !(bytes.Equal(res, bls12381.NewGT().ToBytes(kResult))) { + panic("pairing mismatch blst / geth") + } + return 1 } +func massageBLST(in []byte) []byte { + out := make([]byte, len(in)) + len := 12 * 48 + // 1 + copy(out[0:], in[len-1*48:len]) + copy(out[1*48:], in[len-2*48:len-1*48]) + // 2 + copy(out[6*48:], in[len-3*48:len-2*48]) + copy(out[7*48:], in[len-4*48:len-3*48]) + // 3 + copy(out[2*48:], in[len-5*48:len-4*48]) + copy(out[3*48:], in[len-6*48:len-5*48]) + // 4 + copy(out[8*48:], in[len-7*48:len-6*48]) + copy(out[9*48:], in[len-8*48:len-7*48]) + // 5 + copy(out[4*48:], in[len-9*48:len-8*48]) + copy(out[5*48:], in[len-10*48:len-9*48]) + // 6 + copy(out[10*48:], in[len-11*48:len-10*48]) + copy(out[11*48:], in[len-12*48:len-11*48]) + return out +} + func FuzzCrossG1Add(data []byte) int { input := bytes.NewReader(data) // get random G1 points - kp1, cp1, err := getG1Points(input) + kp1, cp1, bl1, err := getG1Points(input) if err != nil { return 0 } // get random G1 points - kp2, cp2, err := getG1Points(input) + kp2, cp2, bl2, err := getG1Points(input) if err != nil { return 0 } @@ -96,6 +130,11 @@ func FuzzCrossG1Add(data []byte) int { panic("G1 point addition mismatch gnark / geth ") } + bl3 := blst.P1AffinesAdd([]*blst.P1Affine{bl1, bl2}) + if !(bytes.Equal(cp.Marshal(), bl3.Serialize())) { + panic("G1 point addition mismatch blst / geth ") + } + return 1 } @@ -103,13 +142,13 @@ func FuzzCrossG2Add(data []byte) int { input := bytes.NewReader(data) // get random G2 points - kp1, cp1, err := getG2Points(input) + kp1, cp1, bl1, err := getG2Points(input) if err != nil { return 0 } // get random G2 points - kp2, cp2, err := getG2Points(input) + kp2, cp2, bl2, err := getG2Points(input) if err != nil { return 0 } @@ -129,6 +168,11 @@ func FuzzCrossG2Add(data []byte) int { panic("G2 point addition mismatch gnark / geth ") } + bl3 := blst.P2AffinesAdd([]*blst.P2Affine{bl1, bl2}) + if !(bytes.Equal(cp.Marshal(), bl3.Serialize())) { + panic("G1 point addition mismatch blst / geth ") + } + return 1 } @@ -148,7 +192,7 @@ func FuzzCrossG1MultiExp(data []byte) int { break } // get a random G1 point as basis - kp1, cp1, err := getG1Points(input) + kp1, cp1, _, err := getG1Points(input) if err != nil { break } @@ -183,11 +227,11 @@ func FuzzCrossG1MultiExp(data []byte) int { return 1 } -func getG1Points(input io.Reader) (*bls12381.PointG1, *gnark.G1Affine, error) { +func getG1Points(input io.Reader) (*bls12381.PointG1, *gnark.G1Affine, *blst.P1Affine, error) { // sample a random scalar s, err := randomScalar(input, fp.Modulus()) if err != nil { - return nil, nil, err + return nil, nil, nil, err } // compute a random point @@ -206,14 +250,21 @@ func getG1Points(input io.Reader) (*bls12381.PointG1, *gnark.G1Affine, error) { panic("bytes(gnark.G1) != bytes(geth.G1)") } - return kp, cp, nil + // marshal gnark point -> blst point + scalar := new(blst.Scalar).FromBEndian(common.LeftPadBytes(s.Bytes(), 32)) + p1 := new(blst.P1Affine).From(scalar) + if !bytes.Equal(p1.Serialize(), cpBytes) { + panic("bytes(blst.G1) != bytes(geth.G1)") + } + + return kp, cp, p1, nil } -func getG2Points(input io.Reader) (*bls12381.PointG2, *gnark.G2Affine, error) { +func getG2Points(input io.Reader) (*bls12381.PointG2, *gnark.G2Affine, *blst.P2Affine, error) { // sample a random scalar s, err := randomScalar(input, fp.Modulus()) if err != nil { - return nil, nil, err + return nil, nil, nil, err } // compute a random point @@ -232,7 +283,15 @@ func getG2Points(input io.Reader) (*bls12381.PointG2, *gnark.G2Affine, error) { panic("bytes(gnark.G2) != bytes(geth.G2)") } - return kp, cp, nil + // marshal gnark point -> blst point + // Left pad the scalar to 32 bytes + scalar := new(blst.Scalar).FromBEndian(common.LeftPadBytes(s.Bytes(), 32)) + p2 := new(blst.P2Affine).From(scalar) + if !bytes.Equal(p2.Serialize(), cpBytes) { + panic("bytes(blst.G2) != bytes(geth.G2)") + } + + return kp, cp, p2, nil } func randomScalar(r io.Reader, max *big.Int) (k *big.Int, err error) { diff --git a/tests/fuzzers/difficulty/difficulty-fuzz.go b/tests/fuzzers/difficulty/difficulty-fuzz.go index 58936fcd80b10..2112abac1afa3 100644 --- a/tests/fuzzers/difficulty/difficulty-fuzz.go +++ b/tests/fuzzers/difficulty/difficulty-fuzz.go @@ -30,7 +30,6 @@ import ( type fuzzer struct { input io.Reader exhausted bool - debugging bool } func (f *fuzzer) read(size int) []byte { diff --git a/tests/fuzzers/les/les-fuzzer.go b/tests/fuzzers/les/les-fuzzer.go index 3e10171873452..677702208871c 100644 --- a/tests/fuzzers/les/les-fuzzer.go +++ b/tests/fuzzers/les/les-fuzzer.go @@ -88,8 +88,8 @@ func makechain() (bc *core.BlockChain, addrHashes, txHashes []common.Hash) { } func makeTries() (chtTrie *trie.Trie, bloomTrie *trie.Trie, chtKeys, bloomKeys [][]byte) { - chtTrie, _ = trie.New(common.Hash{}, trie.NewDatabase(rawdb.NewMemoryDatabase())) - bloomTrie, _ = trie.New(common.Hash{}, trie.NewDatabase(rawdb.NewMemoryDatabase())) + chtTrie = trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase())) + bloomTrie = trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase())) for i := 0; i < testChainLen; i++ { // The element in CHT is -> key := make([]byte, 8) diff --git a/tests/fuzzers/rangeproof/rangeproof-fuzzer.go b/tests/fuzzers/rangeproof/rangeproof-fuzzer.go index 18717e70d0012..229dea95b1c4a 100644 --- a/tests/fuzzers/rangeproof/rangeproof-fuzzer.go +++ b/tests/fuzzers/rangeproof/rangeproof-fuzzer.go @@ -62,7 +62,7 @@ func (f *fuzzer) readInt() uint64 { } func (f *fuzzer) randomTrie(n int) (*trie.Trie, map[string]*kv) { - trie, _ := trie.New(common.Hash{}, trie.NewDatabase(rawdb.NewMemoryDatabase())) + trie := trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase())) vals := make(map[string]*kv) size := f.readInt() // Fill it with some fluff diff --git a/tests/fuzzers/stacktrie/trie_fuzzer.go b/tests/fuzzers/stacktrie/trie_fuzzer.go index 9ed8bcbc51d58..772c776436f72 100644 --- a/tests/fuzzers/stacktrie/trie_fuzzer.go +++ b/tests/fuzzers/stacktrie/trie_fuzzer.go @@ -25,7 +25,6 @@ import ( "io" "sort" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/trie" "golang.org/x/crypto/sha3" @@ -144,7 +143,7 @@ func (f *fuzzer) fuzz() int { var ( spongeA = &spongeDb{sponge: sha3.NewLegacyKeccak256()} dbA = trie.NewDatabase(spongeA) - trieA, _ = trie.New(common.Hash{}, dbA) + trieA = trie.NewEmpty(dbA) spongeB = &spongeDb{sponge: sha3.NewLegacyKeccak256()} trieB = trie.NewStackTrie(spongeB) vals kvs diff --git a/tests/fuzzers/trie/trie-fuzzer.go b/tests/fuzzers/trie/trie-fuzzer.go index e993af47cf20c..2301721c93113 100644 --- a/tests/fuzzers/trie/trie-fuzzer.go +++ b/tests/fuzzers/trie/trie-fuzzer.go @@ -144,7 +144,7 @@ func runRandTest(rt randTest) error { triedb := trie.NewDatabase(memorydb.New()) - tr, _ := trie.New(common.Hash{}, triedb) + tr := trie.NewEmpty(triedb) values := make(map[string]string) // tracks content of the trie for i, step := range rt { @@ -170,13 +170,13 @@ func runRandTest(rt randTest) error { if err != nil { return err } - newtr, err := trie.New(hash, triedb) + newtr, err := trie.New(common.Hash{}, hash, triedb) if err != nil { return err } tr = newtr case opItercheckhash: - checktr, _ := trie.New(common.Hash{}, triedb) + checktr := trie.NewEmpty(triedb) it := trie.NewIterator(tr.NodeIterator(nil)) for it.Next() { checktr.Update(it.Key, it.Value) diff --git a/tests/init.go b/tests/init.go index 52277e8416423..87ffc65a677c9 100644 --- a/tests/init.go +++ b/tests/init.go @@ -197,6 +197,22 @@ var Forks = map[string]*params.ChainConfig{ LondonBlock: big.NewInt(0), ArrowGlacierBlock: big.NewInt(0), }, + "GrayGlacier": { + ChainID: big.NewInt(1), + HomesteadBlock: big.NewInt(0), + EIP150Block: big.NewInt(0), + EIP155Block: big.NewInt(0), + EIP158Block: big.NewInt(0), + ByzantiumBlock: big.NewInt(0), + ConstantinopleBlock: big.NewInt(0), + PetersburgBlock: big.NewInt(0), + IstanbulBlock: big.NewInt(0), + MuirGlacierBlock: big.NewInt(0), + BerlinBlock: big.NewInt(0), + LondonBlock: big.NewInt(0), + ArrowGlacierBlock: big.NewInt(0), + GrayGlacierBlock: big.NewInt(0), + }, "Merged": { ChainID: big.NewInt(1), HomesteadBlock: big.NewInt(0), @@ -211,7 +227,7 @@ var Forks = map[string]*params.ChainConfig{ BerlinBlock: big.NewInt(0), LondonBlock: big.NewInt(0), ArrowGlacierBlock: big.NewInt(0), - MergeForkBlock: big.NewInt(0), + MergeNetsplitBlock: big.NewInt(0), TerminalTotalDifficulty: big.NewInt(0), }, } diff --git a/tests/init_test.go b/tests/init_test.go index 218634966d83f..4ef5aaf7378fc 100644 --- a/tests/init_test.go +++ b/tests/init_test.go @@ -116,6 +116,7 @@ func (tm *testMatcher) skipLoad(pattern string) { } // fails adds an expected failure for tests matching the pattern. +//nolint:unused func (tm *testMatcher) fails(pattern string, reason string) { if reason == "" { panic("empty fail reason") diff --git a/tests/rlp_test_util.go b/tests/rlp_test_util.go index 9069ec55a15dc..15acb3a244f77 100644 --- a/tests/rlp_test_util.go +++ b/tests/rlp_test_util.go @@ -124,7 +124,7 @@ func translateJSON(v interface{}) interface{} { func checkDecodeFromJSON(s *rlp.Stream, exp interface{}) error { switch exp := exp.(type) { case uint64: - i, err := s.Uint() + i, err := s.Uint64() if err != nil { return addStack("Uint", exp, err) } diff --git a/tests/state_test.go b/tests/state_test.go index d2c92b211cd17..93d8a12106263 100644 --- a/tests/state_test.go +++ b/tests/state_test.go @@ -174,6 +174,7 @@ func runBenchmarkFile(b *testing.B, path string) { return } for _, t := range m { + t := t runBenchmark(b, &t) } } diff --git a/trie/committer.go b/trie/committer.go index 9a7bf48d977fd..9b7ecbf5fcce4 100644 --- a/trie/committer.go +++ b/trie/committer.go @@ -193,9 +193,7 @@ func (c *committer) store(n node, db *Database) node { } else if db != nil { // No leaf-callback used, but there's still a database. Do serial // insertion - db.lock.Lock() db.insert(common.BytesToHash(hash), size, n) - db.lock.Unlock() } return hash } @@ -209,9 +207,7 @@ func (c *committer) commitLoop(db *Database) { n = item.node ) // We are pooling the trie nodes into an intermediate memory cache - db.lock.Lock() db.insert(hash, size, n) - db.lock.Unlock() if c.onleaf != nil { switch n := n.(type) { diff --git a/trie/database.go b/trie/database.go index d71abeee476a5..4167031a584c6 100644 --- a/trie/database.go +++ b/trie/database.go @@ -310,6 +310,9 @@ func (db *Database) DiskDB() ethdb.KeyValueStore { // All nodes inserted by this function will be reference tracked // and in theory should only used for **trie nodes** insertion. func (db *Database) insert(hash common.Hash, size int, node node) { + db.lock.Lock() + defer db.lock.Unlock() + // If the node's already cached, skip if _, ok := db.dirties[hash]; ok { return diff --git a/trie/errors.go b/trie/errors.go index 567b80078c062..afe344bed2692 100644 --- a/trie/errors.go +++ b/trie/errors.go @@ -26,10 +26,21 @@ import ( // in the case where a trie node is not present in the local database. It contains // information necessary for retrieving the missing node. type MissingNodeError struct { + Owner common.Hash // owner of the trie if it's 2-layered trie NodeHash common.Hash // hash of the missing node Path []byte // hex-encoded path to the missing node + err error // concrete error for missing trie node +} + +// Unwrap returns the concrete error for missing trie node which +// allows us for further analysis outside. +func (err *MissingNodeError) Unwrap() error { + return err.err } func (err *MissingNodeError) Error() string { - return fmt.Sprintf("missing trie node %x (path %x)", err.NodeHash, err.Path) + if err.Owner == (common.Hash{}) { + return fmt.Sprintf("missing trie node %x (path %x) %v", err.NodeHash, err.Path, err.err) + } + return fmt.Sprintf("missing trie node %x (owner %x) (path %x) %v", err.NodeHash, err.Owner, err.Path, err.err) } diff --git a/trie/iterator_test.go b/trie/iterator_test.go index ea8a46bb43018..e3e6d0e3a8fa9 100644 --- a/trie/iterator_test.go +++ b/trie/iterator_test.go @@ -297,7 +297,7 @@ func TestUnionIterator(t *testing.T) { } func TestIteratorNoDups(t *testing.T) { - tr, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) + tr := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) for _, val := range testdata1 { tr.Update([]byte(val.k), []byte(val.v)) } @@ -312,7 +312,7 @@ func testIteratorContinueAfterError(t *testing.T, memonly bool) { diskdb := memorydb.New() triedb := NewDatabase(diskdb) - tr, _ := New(common.Hash{}, triedb) + tr := NewEmpty(triedb) for _, val := range testdata1 { tr.Update([]byte(val.k), []byte(val.v)) } @@ -337,7 +337,7 @@ func testIteratorContinueAfterError(t *testing.T, memonly bool) { } for i := 0; i < 20; i++ { // Create trie that will load all nodes from DB. - tr, _ := New(tr.Hash(), triedb) + tr, _ := New(common.Hash{}, tr.Hash(), triedb) // Remove a random node from the database. It can't be the root node // because that one is already loaded. @@ -403,7 +403,7 @@ func testIteratorContinueAfterSeekError(t *testing.T, memonly bool) { diskdb := memorydb.New() triedb := NewDatabase(diskdb) - ctr, _ := New(common.Hash{}, triedb) + ctr := NewEmpty(triedb) for _, val := range testdata1 { ctr.Update([]byte(val.k), []byte(val.v)) } @@ -425,7 +425,7 @@ func testIteratorContinueAfterSeekError(t *testing.T, memonly bool) { } // Create a new iterator that seeks to "bars". Seeking can't proceed because // the node is missing. - tr, _ := New(root, triedb) + tr, _ := New(common.Hash{}, root, triedb) it := tr.NodeIterator([]byte("bars")) missing, ok := it.Error().(*MissingNodeError) if !ok { @@ -513,7 +513,7 @@ func makeLargeTestTrie() (*Database, *SecureTrie, *loggingDb) { // Create an empty trie logDb := &loggingDb{0, memorydb.New()} triedb := NewDatabase(logDb) - trie, _ := NewSecure(common.Hash{}, triedb) + trie, _ := NewSecure(common.Hash{}, common.Hash{}, triedb) // Fill it with some arbitrary data for i := 0; i < 10000; i++ { @@ -546,9 +546,9 @@ func TestNodeIteratorLargeTrie(t *testing.T) { func TestIteratorNodeBlob(t *testing.T) { var ( - db = memorydb.New() - triedb = NewDatabase(db) - trie, _ = New(common.Hash{}, triedb) + db = memorydb.New() + triedb = NewDatabase(db) + trie = NewEmpty(triedb) ) vals := []struct{ k, v string }{ {"do", "verb"}, diff --git a/trie/proof.go b/trie/proof.go index f42dcc761bee7..9bf9107562fa5 100644 --- a/trie/proof.go +++ b/trie/proof.go @@ -367,11 +367,12 @@ func unset(parent node, child node, key []byte, pos int, removeLeft bool) error // branch. The parent must be a fullnode. fn := parent.(*fullNode) fn.Children[key[pos-1]] = nil - } else { - // The key of fork shortnode is greater than the - // path(it doesn't belong to the range), keep - // it with the cached hash available. } + //else { + // The key of fork shortnode is greater than the + // path(it doesn't belong to the range), keep + // it with the cached hash available. + //} } else { if bytes.Compare(cld.Key, key[pos:]) > 0 { // The key of fork shortnode is greater than the @@ -379,11 +380,12 @@ func unset(parent node, child node, key []byte, pos int, removeLeft bool) error // branch. The parent must be a fullnode. fn := parent.(*fullNode) fn.Children[key[pos-1]] = nil - } else { - // The key of fork shortnode is less than the - // path(it doesn't belong to the range), keep - // it with the cached hash available. } + //else { + // The key of fork shortnode is less than the + // path(it doesn't belong to the range), keep + // it with the cached hash available. + //} } return nil } diff --git a/trie/proof_test.go b/trie/proof_test.go index cdf5cf6050980..8db035256e327 100644 --- a/trie/proof_test.go +++ b/trie/proof_test.go @@ -80,7 +80,7 @@ func TestProof(t *testing.T) { } func TestOneElementProof(t *testing.T) { - trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) updateString(trie, "k", "v") for i, prover := range makeProvers(trie) { proof := prover([]byte("k")) @@ -131,7 +131,7 @@ func TestBadProof(t *testing.T) { // Tests that missing keys can also be proven. The test explicitly uses a single // entry trie and checks for missing keys both before and after the single entry. func TestMissingKeyProof(t *testing.T) { - trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) updateString(trie, "k", "v") for i, key := range []string{"a", "j", "l", "z"} { @@ -387,7 +387,7 @@ func TestOneElementRangeProof(t *testing.T) { } // Test the mini trie with only a single element. - tinyTrie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) + tinyTrie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) entry := &kv{randBytes(32), randBytes(20), false} tinyTrie.Update(entry.k, entry.v) @@ -459,7 +459,7 @@ func TestAllElementsProof(t *testing.T) { // TestSingleSideRangeProof tests the range starts from zero. func TestSingleSideRangeProof(t *testing.T) { for i := 0; i < 64; i++ { - trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) var entries entrySlice for i := 0; i < 4096; i++ { value := &kv{randBytes(32), randBytes(20), false} @@ -494,7 +494,7 @@ func TestSingleSideRangeProof(t *testing.T) { // TestReverseSingleSideRangeProof tests the range ends with 0xffff...fff. func TestReverseSingleSideRangeProof(t *testing.T) { for i := 0; i < 64; i++ { - trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) var entries entrySlice for i := 0; i < 4096; i++ { value := &kv{randBytes(32), randBytes(20), false} @@ -601,7 +601,7 @@ func TestBadRangeProof(t *testing.T) { // TestGappedRangeProof focuses on the small trie with embedded nodes. // If the gapped node is embedded in the trie, it should be detected too. func TestGappedRangeProof(t *testing.T) { - trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) var entries []*kv // Sorted entries for i := byte(0); i < 10; i++ { value := &kv{common.LeftPadBytes([]byte{i}, 32), []byte{i}, false} @@ -675,7 +675,7 @@ func TestSameSideProofs(t *testing.T) { } func TestHasRightElement(t *testing.T) { - trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) var entries entrySlice for i := 0; i < 4096; i++ { value := &kv{randBytes(32), randBytes(20), false} @@ -1028,7 +1028,7 @@ func benchmarkVerifyRangeNoProof(b *testing.B, size int) { } func randomTrie(n int) (*Trie, map[string]*kv) { - trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) vals := make(map[string]*kv) for i := byte(0); i < 100; i++ { value := &kv{common.LeftPadBytes([]byte{i}, 32), []byte{i}, false} @@ -1053,7 +1053,7 @@ func randBytes(n int) []byte { } func nonRandomTrie(n int) (*Trie, map[string]*kv) { - trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) vals := make(map[string]*kv) max := uint64(0xffffffffffffffff) for i := uint64(0); i < uint64(n); i++ { @@ -1078,7 +1078,7 @@ func TestRangeProofKeysWithSharedPrefix(t *testing.T) { common.Hex2Bytes("02"), common.Hex2Bytes("03"), } - trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) for i, key := range keys { trie.Update(key, vals[i]) } diff --git a/trie/secure_trie.go b/trie/secure_trie.go index 248b93544d2fe..6a5cc89c9ffd2 100644 --- a/trie/secure_trie.go +++ b/trie/secure_trie.go @@ -53,11 +53,11 @@ type SecureTrie struct { // Loaded nodes are kept around until their 'cache generation' expires. // A new cache generation is created by each call to Commit. // cachelimit sets the number of past cache generations to keep. -func NewSecure(root common.Hash, db *Database) (*SecureTrie, error) { +func NewSecure(owner common.Hash, root common.Hash, db *Database) (*SecureTrie, error) { if db == nil { panic("trie.NewSecure called without a database") } - trie, err := New(root, db) + trie, err := New(owner, root, db) if err != nil { return nil, err } diff --git a/trie/secure_trie_test.go b/trie/secure_trie_test.go index a3ece84b57120..beea5845ad0d6 100644 --- a/trie/secure_trie_test.go +++ b/trie/secure_trie_test.go @@ -28,7 +28,7 @@ import ( ) func newEmptySecure() *SecureTrie { - trie, _ := NewSecure(common.Hash{}, NewDatabase(memorydb.New())) + trie, _ := NewSecure(common.Hash{}, common.Hash{}, NewDatabase(memorydb.New())) return trie } @@ -36,7 +36,7 @@ func newEmptySecure() *SecureTrie { func makeTestSecureTrie() (*Database, *SecureTrie, map[string][]byte) { // Create an empty trie triedb := NewDatabase(memorydb.New()) - trie, _ := NewSecure(common.Hash{}, triedb) + trie, _ := NewSecure(common.Hash{}, common.Hash{}, triedb) // Fill it with some arbitrary data content := make(map[string][]byte) diff --git a/trie/stacktrie.go b/trie/stacktrie.go index b38bb01b0fb32..a22fa0dd6797f 100644 --- a/trie/stacktrie.go +++ b/trie/stacktrie.go @@ -38,9 +38,10 @@ var stPool = sync.Pool{ }, } -func stackTrieFromPool(db ethdb.KeyValueWriter) *StackTrie { +func stackTrieFromPool(db ethdb.KeyValueWriter, owner common.Hash) *StackTrie { st := stPool.Get().(*StackTrie) st.db = db + st.owner = owner return st } @@ -53,6 +54,7 @@ func returnToPool(st *StackTrie) { // in order. Once it determines that a subtree will no longer be inserted // into, it will hash it and free up the memory it uses. type StackTrie struct { + owner common.Hash // the owner of the trie nodeType uint8 // node type (as in branch, ext, leaf) val []byte // value contained by this node if it's a leaf key []byte // key chunk covered by this (leaf|ext) node @@ -68,6 +70,16 @@ func NewStackTrie(db ethdb.KeyValueWriter) *StackTrie { } } +// NewStackTrieWithOwner allocates and initializes an empty trie, but with +// the additional owner field. +func NewStackTrieWithOwner(db ethdb.KeyValueWriter, owner common.Hash) *StackTrie { + return &StackTrie{ + owner: owner, + nodeType: emptyNode, + db: db, + } +} + // NewFromBinary initialises a serialized stacktrie with the given db. func NewFromBinary(data []byte, db ethdb.KeyValueWriter) (*StackTrie, error) { var st StackTrie @@ -88,10 +100,12 @@ func (st *StackTrie) MarshalBinary() (data []byte, err error) { w = bufio.NewWriter(&b) ) if err := gob.NewEncoder(w).Encode(struct { - Nodetype uint8 + Owner common.Hash + NodeType uint8 Val []byte Key []byte }{ + st.owner, st.nodeType, st.val, st.key, @@ -122,12 +136,14 @@ func (st *StackTrie) UnmarshalBinary(data []byte) error { func (st *StackTrie) unmarshalBinary(r io.Reader) error { var dec struct { - Nodetype uint8 + Owner common.Hash + NodeType uint8 Val []byte Key []byte } gob.NewDecoder(r).Decode(&dec) - st.nodeType = dec.Nodetype + st.owner = dec.Owner + st.nodeType = dec.NodeType st.val = dec.Val st.key = dec.Key @@ -154,16 +170,16 @@ func (st *StackTrie) setDb(db ethdb.KeyValueWriter) { } } -func newLeaf(key, val []byte, db ethdb.KeyValueWriter) *StackTrie { - st := stackTrieFromPool(db) +func newLeaf(owner common.Hash, key, val []byte, db ethdb.KeyValueWriter) *StackTrie { + st := stackTrieFromPool(db, owner) st.nodeType = leafNode st.key = append(st.key, key...) st.val = val return st } -func newExt(key []byte, child *StackTrie, db ethdb.KeyValueWriter) *StackTrie { - st := stackTrieFromPool(db) +func newExt(owner common.Hash, key []byte, child *StackTrie, db ethdb.KeyValueWriter) *StackTrie { + st := stackTrieFromPool(db, owner) st.nodeType = extNode st.key = append(st.key, key...) st.children[0] = child @@ -196,6 +212,7 @@ func (st *StackTrie) Update(key, value []byte) { } func (st *StackTrie) Reset() { + st.owner = common.Hash{} st.db = nil st.key = st.key[:0] st.val = nil @@ -236,7 +253,7 @@ func (st *StackTrie) insert(key, value []byte) { // Add new child if st.children[idx] == nil { - st.children[idx] = newLeaf(key[1:], value, st.db) + st.children[idx] = newLeaf(st.owner, key[1:], value, st.db) } else { st.children[idx].insert(key[1:], value) } @@ -262,7 +279,7 @@ func (st *StackTrie) insert(key, value []byte) { // node directly. var n *StackTrie if diffidx < len(st.key)-1 { - n = newExt(st.key[diffidx+1:], st.children[0], st.db) + n = newExt(st.owner, st.key[diffidx+1:], st.children[0], st.db) } else { // Break on the last byte, no need to insert // an extension node: reuse the current node @@ -282,12 +299,12 @@ func (st *StackTrie) insert(key, value []byte) { // the common prefix is at least one byte // long, insert a new intermediate branch // node. - st.children[0] = stackTrieFromPool(st.db) + st.children[0] = stackTrieFromPool(st.db, st.owner) st.children[0].nodeType = branchNode p = st.children[0] } // Create a leaf for the inserted part - o := newLeaf(key[diffidx+1:], value, st.db) + o := newLeaf(st.owner, key[diffidx+1:], value, st.db) // Insert both child leaves where they belong: origIdx := st.key[diffidx] @@ -323,7 +340,7 @@ func (st *StackTrie) insert(key, value []byte) { // Convert current node into an ext, // and insert a child branch node. st.nodeType = extNode - st.children[0] = NewStackTrie(st.db) + st.children[0] = NewStackTrieWithOwner(st.db, st.owner) st.children[0].nodeType = branchNode p = st.children[0] } @@ -332,11 +349,11 @@ func (st *StackTrie) insert(key, value []byte) { // value and another containing the new value. The child leaf // is hashed directly in order to free up some memory. origIdx := st.key[diffidx] - p.children[origIdx] = newLeaf(st.key[diffidx+1:], st.val, st.db) + p.children[origIdx] = newLeaf(st.owner, st.key[diffidx+1:], st.val, st.db) p.children[origIdx].hash() newIdx := key[diffidx] - p.children[newIdx] = newLeaf(key[diffidx+1:], value, st.db) + p.children[newIdx] = newLeaf(st.owner, key[diffidx+1:], value, st.db) // Finally, cut off the key part that has been passed // over to the children. diff --git a/trie/stacktrie_test.go b/trie/stacktrie_test.go index e57df60369bf2..f24c749716e50 100644 --- a/trie/stacktrie_test.go +++ b/trie/stacktrie_test.go @@ -188,7 +188,7 @@ func TestStackTrieInsertAndHash(t *testing.T) { func TestSizeBug(t *testing.T) { st := NewStackTrie(nil) - nt, _ := New(common.Hash{}, NewDatabase(memorydb.New())) + nt := NewEmpty(NewDatabase(memorydb.New())) leaf := common.FromHex("290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563") value := common.FromHex("94cf40d0d2b44f2b66e07cace1372ca42b73cf21a3") @@ -203,7 +203,7 @@ func TestSizeBug(t *testing.T) { func TestEmptyBug(t *testing.T) { st := NewStackTrie(nil) - nt, _ := New(common.Hash{}, NewDatabase(memorydb.New())) + nt := NewEmpty(NewDatabase(memorydb.New())) //leaf := common.FromHex("290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563") //value := common.FromHex("94cf40d0d2b44f2b66e07cace1372ca42b73cf21a3") @@ -229,7 +229,7 @@ func TestEmptyBug(t *testing.T) { func TestValLength56(t *testing.T) { st := NewStackTrie(nil) - nt, _ := New(common.Hash{}, NewDatabase(memorydb.New())) + nt := NewEmpty(NewDatabase(memorydb.New())) //leaf := common.FromHex("290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563") //value := common.FromHex("94cf40d0d2b44f2b66e07cace1372ca42b73cf21a3") @@ -254,7 +254,8 @@ func TestValLength56(t *testing.T) { // which causes a lot of node-within-node. This case was found via fuzzing. func TestUpdateSmallNodes(t *testing.T) { st := NewStackTrie(nil) - nt, _ := New(common.Hash{}, NewDatabase(memorydb.New())) + nt := NewEmpty(NewDatabase(memorydb.New())) + kvs := []struct { K string V string @@ -282,7 +283,8 @@ func TestUpdateSmallNodes(t *testing.T) { func TestUpdateVariableKeys(t *testing.T) { t.SkipNow() st := NewStackTrie(nil) - nt, _ := New(common.Hash{}, NewDatabase(memorydb.New())) + nt := NewEmpty(NewDatabase(memorydb.New())) + kvs := []struct { K string V string @@ -352,7 +354,7 @@ func TestStacktrieNotModifyValues(t *testing.T) { func TestStacktrieSerialization(t *testing.T) { var ( st = NewStackTrie(nil) - nt, _ = New(common.Hash{}, NewDatabase(memorydb.New())) + nt = NewEmpty(NewDatabase(memorydb.New())) keyB = big.NewInt(1) keyDelta = big.NewInt(1) vals [][]byte diff --git a/trie/sync_test.go b/trie/sync_test.go index 970730b671870..4c2c50d7a1597 100644 --- a/trie/sync_test.go +++ b/trie/sync_test.go @@ -29,7 +29,7 @@ import ( func makeTestTrie() (*Database, *SecureTrie, map[string][]byte) { // Create an empty trie triedb := NewDatabase(memorydb.New()) - trie, _ := NewSecure(common.Hash{}, triedb) + trie, _ := NewSecure(common.Hash{}, common.Hash{}, triedb) // Fill it with some arbitrary data content := make(map[string][]byte) @@ -60,7 +60,7 @@ func makeTestTrie() (*Database, *SecureTrie, map[string][]byte) { // content map. func checkTrieContents(t *testing.T, db *Database, root []byte, content map[string][]byte) { // Check root availability and trie contents - trie, err := NewSecure(common.BytesToHash(root), db) + trie, err := NewSecure(common.Hash{}, common.BytesToHash(root), db) if err != nil { t.Fatalf("failed to create trie at %x: %v", root, err) } @@ -77,7 +77,7 @@ func checkTrieContents(t *testing.T, db *Database, root []byte, content map[stri // checkTrieConsistency checks that all nodes in a trie are indeed present. func checkTrieConsistency(db *Database, root common.Hash) error { // Create and iterate a trie rooted in a subnode - trie, err := NewSecure(root, db) + trie, err := NewSecure(common.Hash{}, root, db) if err != nil { return nil // Consider a non existent state consistent } @@ -91,8 +91,8 @@ func checkTrieConsistency(db *Database, root common.Hash) error { func TestEmptySync(t *testing.T) { dbA := NewDatabase(memorydb.New()) dbB := NewDatabase(memorydb.New()) - emptyA, _ := New(common.Hash{}, dbA) - emptyB, _ := New(emptyRoot, dbB) + emptyA := NewEmpty(dbA) + emptyB, _ := New(common.Hash{}, emptyRoot, dbB) for i, trie := range []*Trie{emptyA, emptyB} { sync := NewSync(trie.Hash(), memorydb.New(), nil) diff --git a/trie/trie.go b/trie/trie.go index fe7d6dc17e79f..0c81cb2c39019 100644 --- a/trie/trie.go +++ b/trie/trie.go @@ -61,8 +61,9 @@ type LeafCallback func(paths [][]byte, hexpath []byte, leaf []byte, parent commo // // Trie is not safe for concurrent use. type Trie struct { - db *Database - root node + db *Database + root node + owner common.Hash // Keep track of the number leaves which have been inserted since the last // hashing operation. This number will not directly map to the number of @@ -84,23 +85,47 @@ func (t *Trie) Copy() *Trie { return &Trie{ db: t.db, root: t.root, + owner: t.owner, unhashed: t.unhashed, tracer: t.tracer.copy(), } } -// New creates a trie with an existing root node from db. +// New creates a trie with an existing root node from db and an assigned +// owner for storage proximity. // // If root is the zero hash or the sha3 hash of an empty string, the // trie is initially empty and does not require a database. Otherwise, // New will panic if db is nil and returns a MissingNodeError if root does // not exist in the database. Accessing the trie loads nodes from db on demand. -func New(root common.Hash, db *Database) (*Trie, error) { +func New(owner common.Hash, root common.Hash, db *Database) (*Trie, error) { + return newTrie(owner, root, db) +} + +// NewEmpty is a shortcut to create empty tree. It's mostly used in tests. +func NewEmpty(db *Database) *Trie { + tr, _ := newTrie(common.Hash{}, common.Hash{}, db) + return tr +} + +// newWithRootNode initializes the trie with the given root node. +// It's only used by range prover. +func newWithRootNode(root node) *Trie { + return &Trie{ + root: root, + //tracer: newTracer(), + db: NewDatabase(rawdb.NewMemoryDatabase()), + } +} + +// newTrie is the internal function used to construct the trie with given parameters. +func newTrie(owner common.Hash, root common.Hash, db *Database) (*Trie, error) { if db == nil { panic("trie.New called without a database") } trie := &Trie{ - db: db, + db: db, + owner: owner, //tracer: newTracer(), } if root != (common.Hash{}) && root != emptyRoot { @@ -113,16 +138,6 @@ func New(root common.Hash, db *Database) (*Trie, error) { return trie, nil } -// newWithRootNode initializes the trie with the given root node. -// It's only used by range prover. -func newWithRootNode(root node) *Trie { - return &Trie{ - root: root, - //tracer: newTracer(), - db: NewDatabase(rawdb.NewMemoryDatabase()), - } -} - // NodeIterator returns an iterator that returns nodes of the trie. Iteration starts at // the key after the given start key. func (t *Trie) NodeIterator(start []byte) NodeIterator { @@ -562,7 +577,7 @@ func (t *Trie) resolveHash(n hashNode, prefix []byte) (node, error) { if node := t.db.node(hash); node != nil { return node, nil } - return nil, &MissingNodeError{NodeHash: hash, Path: prefix} + return nil, &MissingNodeError{Owner: t.owner, NodeHash: hash, Path: prefix} } func (t *Trie) resolveBlob(n hashNode, prefix []byte) ([]byte, error) { @@ -571,7 +586,7 @@ func (t *Trie) resolveBlob(n hashNode, prefix []byte) ([]byte, error) { if len(blob) != 0 { return blob, nil } - return nil, &MissingNodeError{NodeHash: hash, Path: prefix} + return nil, &MissingNodeError{Owner: t.owner, NodeHash: hash, Path: prefix} } // Hash returns the root hash of the trie. It does not write to the @@ -602,7 +617,10 @@ func (t *Trie) Commit(onleaf LeafCallback) (common.Hash, int, error) { // Do a quick check if we really need to commit, before we spin // up goroutines. This can happen e.g. if we load a trie for reading storage // values, but don't write to it. - if _, dirty := t.root.cache(); !dirty { + if hashedNode, dirty := t.root.cache(); !dirty { + // Replace the root node with the origin hash in order to + // ensure all resolved nodes are dropped after the commit. + t.root = hashedNode return rootHash, 0, nil } var wg sync.WaitGroup @@ -647,6 +665,12 @@ func (t *Trie) hashRoot() (node, node, error) { // Reset drops the referenced root node and cleans all internal state. func (t *Trie) Reset() { t.root = nil + t.owner = common.Hash{} t.unhashed = 0 t.tracer.reset() } + +// Owner returns the associated trie owner. +func (t *Trie) Owner() common.Hash { + return t.owner +} diff --git a/trie/trie_test.go b/trie/trie_test.go index f994e31af40e1..ab65965b4972c 100644 --- a/trie/trie_test.go +++ b/trie/trie_test.go @@ -48,12 +48,12 @@ func init() { // Used for testing func newEmpty() *Trie { - trie, _ := New(common.Hash{}, NewDatabase(memorydb.New())) + trie := NewEmpty(NewDatabase(memorydb.New())) return trie } func TestEmptyTrie(t *testing.T) { - trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) res := trie.Hash() exp := emptyRoot if res != exp { @@ -62,7 +62,7 @@ func TestEmptyTrie(t *testing.T) { } func TestNull(t *testing.T) { - trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) key := make([]byte, 32) value := []byte("test") trie.Update(key, value) @@ -72,7 +72,7 @@ func TestNull(t *testing.T) { } func TestMissingRoot(t *testing.T) { - trie, err := New(common.HexToHash("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33"), NewDatabase(memorydb.New())) + trie, err := New(common.Hash{}, common.HexToHash("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33"), NewDatabase(memorydb.New())) if trie != nil { t.Error("New returned non-nil trie for invalid root") } @@ -88,7 +88,7 @@ func testMissingNode(t *testing.T, memonly bool) { diskdb := memorydb.New() triedb := NewDatabase(diskdb) - trie, _ := New(common.Hash{}, triedb) + trie := NewEmpty(triedb) updateString(trie, "120000", "qwerqwerqwerqwerqwerqwerqwerqwer") updateString(trie, "123456", "asdfasdfasdfasdfasdfasdfasdfasdf") root, _, _ := trie.Commit(nil) @@ -96,27 +96,27 @@ func testMissingNode(t *testing.T, memonly bool) { triedb.Commit(root, true, nil) } - trie, _ = New(root, triedb) + trie, _ = New(common.Hash{}, root, triedb) _, err := trie.TryGet([]byte("120000")) if err != nil { t.Errorf("Unexpected error: %v", err) } - trie, _ = New(root, triedb) + trie, _ = New(common.Hash{}, root, triedb) _, err = trie.TryGet([]byte("120099")) if err != nil { t.Errorf("Unexpected error: %v", err) } - trie, _ = New(root, triedb) + trie, _ = New(common.Hash{}, root, triedb) _, err = trie.TryGet([]byte("123456")) if err != nil { t.Errorf("Unexpected error: %v", err) } - trie, _ = New(root, triedb) + trie, _ = New(common.Hash{}, root, triedb) err = trie.TryUpdate([]byte("120099"), []byte("zxcvzxcvzxcvzxcvzxcvzxcvzxcvzxcv")) if err != nil { t.Errorf("Unexpected error: %v", err) } - trie, _ = New(root, triedb) + trie, _ = New(common.Hash{}, root, triedb) err = trie.TryDelete([]byte("123456")) if err != nil { t.Errorf("Unexpected error: %v", err) @@ -129,27 +129,27 @@ func testMissingNode(t *testing.T, memonly bool) { diskdb.Delete(hash[:]) } - trie, _ = New(root, triedb) + trie, _ = New(common.Hash{}, root, triedb) _, err = trie.TryGet([]byte("120000")) if _, ok := err.(*MissingNodeError); !ok { t.Errorf("Wrong error: %v", err) } - trie, _ = New(root, triedb) + trie, _ = New(common.Hash{}, root, triedb) _, err = trie.TryGet([]byte("120099")) if _, ok := err.(*MissingNodeError); !ok { t.Errorf("Wrong error: %v", err) } - trie, _ = New(root, triedb) + trie, _ = New(common.Hash{}, root, triedb) _, err = trie.TryGet([]byte("123456")) if err != nil { t.Errorf("Unexpected error: %v", err) } - trie, _ = New(root, triedb) + trie, _ = New(common.Hash{}, root, triedb) err = trie.TryUpdate([]byte("120099"), []byte("zxcv")) if _, ok := err.(*MissingNodeError); !ok { t.Errorf("Wrong error: %v", err) } - trie, _ = New(root, triedb) + trie, _ = New(common.Hash{}, root, triedb) err = trie.TryDelete([]byte("123456")) if _, ok := err.(*MissingNodeError); !ok { t.Errorf("Wrong error: %v", err) @@ -277,7 +277,7 @@ func TestReplication(t *testing.T) { } // create a new trie on top of the database and check that lookups work. - trie2, err := New(exp, trie.db) + trie2, err := New(common.Hash{}, exp, trie.db) if err != nil { t.Fatalf("can't recreate trie at %x: %v", exp, err) } @@ -410,10 +410,10 @@ func (randTest) Generate(r *rand.Rand, size int) reflect.Value { func runRandTest(rt randTest) bool { var ( - triedb = NewDatabase(memorydb.New()) - tr, _ = New(common.Hash{}, triedb) - values = make(map[string]string) // tracks content of the trie - origTrie, _ = New(common.Hash{}, triedb) + triedb = NewDatabase(memorydb.New()) + tr = NewEmpty(triedb) + values = make(map[string]string) // tracks content of the trie + origTrie = NewEmpty(triedb) ) tr.tracer = newTracer() @@ -445,7 +445,7 @@ func runRandTest(rt randTest) bool { rt[i].err = err return false } - newtr, err := New(hash, triedb) + newtr, err := New(common.Hash{}, hash, triedb) if err != nil { rt[i].err = err return false @@ -455,7 +455,7 @@ func runRandTest(rt randTest) bool { origTrie = tr.Copy() case opItercheckhash: - checktr, _ := New(common.Hash{}, triedb) + checktr := NewEmpty(triedb) it := NewIterator(tr.NodeIterator(nil)) for it.Next() { checktr.Update(it.Key, it.Value) @@ -542,10 +542,10 @@ func BenchmarkUpdateLE(b *testing.B) { benchUpdate(b, binary.LittleEndian) } const benchElemCount = 20000 func benchGet(b *testing.B, commit bool) { - trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) if commit { tmpdb := tempDB(b) - trie, _ = New(common.Hash{}, tmpdb) + trie = NewEmpty(tmpdb) } k := make([]byte, 32) for i := 0; i < benchElemCount; i++ { @@ -664,7 +664,7 @@ func TestTinyTrie(t *testing.T) { if exp, root := common.HexToHash("0608c1d1dc3905fa22204c7a0e43644831c3b6d3def0f274be623a948197e64a"), trie.Hash(); exp != root { t.Errorf("3: got %x, exp %x", root, exp) } - checktr, _ := New(common.Hash{}, trie.db) + checktr := NewEmpty(trie.db) it := NewIterator(trie.NodeIterator(nil)) for it.Next() { checktr.Update(it.Key, it.Value) @@ -790,7 +790,7 @@ func TestCommitSequence(t *testing.T) { // This spongeDb is used to check the sequence of disk-db-writes s := &spongeDb{sponge: sha3.NewLegacyKeccak256()} db := NewDatabase(s) - trie, _ := New(common.Hash{}, db) + trie := NewEmpty(db) // Another sponge is used to check the callback-sequence callbackSponge := sha3.NewLegacyKeccak256() // Fill the trie with elements @@ -832,7 +832,7 @@ func TestCommitSequenceRandomBlobs(t *testing.T) { // This spongeDb is used to check the sequence of disk-db-writes s := &spongeDb{sponge: sha3.NewLegacyKeccak256()} db := NewDatabase(s) - trie, _ := New(common.Hash{}, db) + trie := NewEmpty(db) // Another sponge is used to check the callback-sequence callbackSponge := sha3.NewLegacyKeccak256() // Fill the trie with elements @@ -871,7 +871,7 @@ func TestCommitSequenceStackTrie(t *testing.T) { // This spongeDb is used to check the sequence of disk-db-writes s := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "a"} db := NewDatabase(s) - trie, _ := New(common.Hash{}, db) + trie := NewEmpty(db) // Another sponge is used for the stacktrie commits stackTrieSponge := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "b"} stTrie := NewStackTrie(stackTrieSponge) @@ -927,7 +927,7 @@ func TestCommitSequenceStackTrie(t *testing.T) { func TestCommitSequenceSmallRoot(t *testing.T) { s := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "a"} db := NewDatabase(s) - trie, _ := New(common.Hash{}, db) + trie := NewEmpty(db) // Another sponge is used for the stacktrie commits stackTrieSponge := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "b"} stTrie := NewStackTrie(stackTrieSponge) diff --git a/trie/util_test.go b/trie/util_test.go index fadb0553b5298..589eca62423ac 100644 --- a/trie/util_test.go +++ b/trie/util_test.go @@ -19,14 +19,12 @@ package trie import ( "testing" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" ) // Tests if the trie diffs are tracked correctly. func TestTrieTracer(t *testing.T) { - db := NewDatabase(rawdb.NewMemoryDatabase()) - trie, _ := New(common.Hash{}, db) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) trie.tracer = newTracer() // Insert a batch of entries, all the nodes should be marked as inserted @@ -93,8 +91,7 @@ func TestTrieTracer(t *testing.T) { } func TestTrieTracerNoop(t *testing.T) { - db := NewDatabase(rawdb.NewMemoryDatabase()) - trie, _ := New(common.Hash{}, db) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) trie.tracer = newTracer() // Insert a batch of entries, all the nodes should be marked as inserted diff --git a/trie/utils.go b/trie/utils.go index 5f9e3ba58ecc3..7e26915041efd 100644 --- a/trie/utils.go +++ b/trie/utils.go @@ -29,24 +29,41 @@ package trie // This tool can track all of them no matter the node is embedded in its // parent or not, but valueNode is never tracked. // +// Besides, it's also used for recording the original value of the nodes +// when they are resolved from the disk. The pre-value of the nodes will +// be used to construct reverse-diffs in the future. +// // Note tracer is not thread-safe, callers should be responsible for handling // the concurrency issues by themselves. type tracer struct { insert map[string]struct{} delete map[string]struct{} + origin map[string][]byte } -// newTracer initializes trie node diff tracer. +// newTracer initializes the tracer for capturing trie changes. func newTracer() *tracer { return &tracer{ insert: make(map[string]struct{}), delete: make(map[string]struct{}), + origin: make(map[string][]byte), + } +} + +/* +// onRead tracks the newly loaded trie node and caches the rlp-encoded blob internally. +// Don't change the value outside of function since it's not deep-copied. +func (t *tracer) onRead(key []byte, val []byte) { + // Tracer isn't used right now, remove this check later. + if t == nil { + return } + t.origin[string(key)] = val } +*/ -// onInsert tracks the newly inserted trie node. If it's already -// in the deletion set(resurrected node), then just wipe it from -// the deletion set as it's untouched. +// onInsert tracks the newly inserted trie node. If it's already in the deletion set +// (resurrected node), then just wipe it from the deletion set as the "untouched". func (t *tracer) onInsert(key []byte) { // Tracer isn't used right now, remove this check later. if t == nil { @@ -100,6 +117,17 @@ func (t *tracer) deleteList() [][]byte { return ret } +/* +// getPrev returns the cached original value of the specified node. +func (t *tracer) getPrev(key []byte) []byte { + // Don't panic on uninitialized tracer, it's possible in testing. + if t == nil { + return nil + } + return t.origin[string(key)] +} +*/ + // reset clears the content tracked by tracer. func (t *tracer) reset() { // Tracer isn't used right now, remove this check later. @@ -108,6 +136,7 @@ func (t *tracer) reset() { } t.insert = make(map[string]struct{}) t.delete = make(map[string]struct{}) + t.origin = make(map[string][]byte) } // copy returns a deep copied tracer instance. @@ -119,6 +148,7 @@ func (t *tracer) copy() *tracer { var ( insert = make(map[string]struct{}) delete = make(map[string]struct{}) + origin = make(map[string][]byte) ) for key := range t.insert { insert[key] = struct{}{} @@ -126,8 +156,12 @@ func (t *tracer) copy() *tracer { for key := range t.delete { delete[key] = struct{}{} } + for key, val := range t.origin { + origin[key] = val + } return &tracer{ insert: insert, delete: delete, + origin: origin, } }