diff --git a/.golangci.yml b/.golangci.yml index 4950b98c21baf..4c12972235331 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -19,10 +19,25 @@ linters: - govet - ineffassign - misspell - # - staticcheck - unconvert - # - unused - varcheck + - typecheck + - unused + - staticcheck + - bidichk + - durationcheck + - exportloopref + - gosec + - whitespace + + # - structcheck # lots of false positives + # - errcheck #lot of false positives + # - contextcheck + # - errchkjson # lots of false positives + # - errorlint # this check crashes + # - exhaustive # silly check + # - makezero # false positives + # - nilerr # several intentional linters-settings: gofmt: @@ -30,21 +45,29 @@ linters-settings: goconst: min-len: 3 # minimum length of string constant min-occurrences: 6 # minimum number of occurrences + gosec: + excludes: + - G404 # Use of weak random number generator - lots of FP + - G107 # Potential http request -- those are intentional + - G306 # G306: Expect WriteFile permissions to be 0600 or less issues: exclude-rules: - - path: crypto/blake2b/ - linters: - - deadcode - - path: crypto/bn256/cloudflare - linters: - - deadcode - - path: p2p/discv5/ - linters: - - deadcode - - path: core/vm/instructions_test.go - linters: - - goconst - - path: cmd/faucet/ + - path: crypto/bn256/cloudflare/optate.go linters: - deadcode + - staticcheck + - path: internal/build/pgp.go + text: 'SA1019: package golang.org/x/crypto/openpgp is deprecated' + - path: core/vm/contracts.go + text: 'SA1019: package golang.org/x/crypto/ripemd160 is deprecated' + - path: accounts/usbwallet/trezor.go + text: 'SA1019: package github.com/golang/protobuf/proto is deprecated' + - path: accounts/usbwallet/trezor/ + text: 'SA1019: package github.com/golang/protobuf/proto is deprecated' + exclude: + - 'SA1019: event.TypeMux is deprecated: use Feed' + - 'SA1019: strings.Title is deprecated' + - 'SA1019: strings.Title has been deprecated since Go 1.18 and an alternative has been available since Go 1.0: The rule Title uses for word boundaries does not handle Unicode punctuation properly. Use golang.org/x/text/cases instead.' + - 'SA1029: should not use built-in type string as key for value' + - 'G306: Expect WriteFile permissions to be 0600 or less' diff --git a/Dockerfile b/Dockerfile index 70299190f90f5..143c92f27f505 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ COPY go.sum /go-ethereum/ RUN cd /go-ethereum && go mod download ADD . /go-ethereum -RUN cd /go-ethereum && go run build/ci.go install ./cmd/geth +RUN cd /go-ethereum && go run build/ci.go install -static ./cmd/geth # Pull Geth into a second stage deploy alpine container FROM alpine:latest diff --git a/Dockerfile.alltools b/Dockerfile.alltools index b11492cabc9c0..176c4592206d5 100644 --- a/Dockerfile.alltools +++ b/Dockerfile.alltools @@ -14,7 +14,7 @@ COPY go.sum /go-ethereum/ RUN cd /go-ethereum && go mod download ADD . /go-ethereum -RUN cd /go-ethereum && go run build/ci.go install +RUN cd /go-ethereum && go run build/ci.go install -static # Pull all binaries into a second stage deploy alpine container FROM alpine:latest diff --git a/README.md b/README.md index 0987200d3b9a2..b20eb5b748b4b 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ directory. | `abigen` | Source code generator to convert Ethereum contract definitions into easy to use, compile-time type-safe Go packages. It operates on plain [Ethereum contract ABIs](https://docs.soliditylang.org/en/develop/abi-spec.html) with expanded functionality if the contract bytecode is also available. However, it also accepts Solidity source files, making development much more streamlined. Please see our [Native DApps](https://geth.ethereum.org/docs/dapp/native-bindings) page for details. | | `bootnode` | Stripped down version of our Ethereum client implementation that only takes part in the network node discovery protocol, but does not run any of the higher level application protocols. It can be used as a lightweight bootstrap node to aid in finding peers in private networks. | | `evm` | Developer utility version of the EVM (Ethereum Virtual Machine) that is capable of running bytecode snippets within a configurable environment and execution mode. Its purpose is to allow isolated, fine-grained debugging of EVM opcodes (e.g. `evm --code 60ff60ff --debug run`). | -| `rlpdump` | Developer utility tool to convert binary RLP ([Recursive Length Prefix](https://eth.wiki/en/fundamentals/rlp)) dumps (data encoding used by the Ethereum protocol both network as well as consensus wise) to user-friendlier hierarchical representation (e.g. `rlpdump --hex CE0183FFFFFFC4C304050583616263`). | +| `rlpdump` | Developer utility tool to convert binary RLP ([Recursive Length Prefix](https://ethereum.org/en/developers/docs/data-structures-and-encoding/rlp)) dumps (data encoding used by the Ethereum protocol both network as well as consensus wise) to user-friendlier hierarchical representation (e.g. `rlpdump --hex CE0183FFFFFFC4C304050583616263`). | | `puppeth` | a CLI wizard that aids in creating a new Ethereum network. | ## Running `geth` @@ -188,7 +188,7 @@ accessible from the outside. As a developer, sooner rather than later you'll want to start interacting with `geth` and the Ethereum network via your own programs and not manually through the console. To aid -this, `geth` has built-in support for a JSON-RPC based APIs ([standard APIs](https://eth.wiki/json-rpc/API) +this, `geth` has built-in support for a JSON-RPC based APIs ([standard APIs](https://ethereum.github.io/execution-apis/api-documentation/) and [`geth` specific APIs](https://geth.ethereum.org/docs/rpc/server)). These can be exposed via HTTP, WebSockets and IPC (UNIX sockets on UNIX based platforms, and named pipes on Windows). @@ -211,7 +211,7 @@ HTTP based JSON-RPC API options: * `--ws.api` API's offered over the WS-RPC interface (default: `eth,net,web3`) * `--ws.origins` Origins from which to accept websockets requests * `--ipcdisable` Disable the IPC-RPC server - * `--ipcapi` API's offered over the IPC-RPC interface (default: `admin,debug,eth,miner,net,personal,shh,txpool,web3`) + * `--ipcapi` API's offered over the IPC-RPC interface (default: `admin,debug,eth,miner,net,personal,txpool,web3`) * `--ipcpath` Filename for IPC socket/pipe within the datadir (explicit paths escape it) You'll need to use your own programming environments' capabilities (libraries, tools, etc) to @@ -297,7 +297,7 @@ $ bootnode --genkey=boot.key $ bootnode --nodekey=boot.key ``` -With the bootnode online, it will display an [`enode` URL](https://eth.wiki/en/fundamentals/enode-url-format) +With the bootnode online, it will display an [`enode` URL](https://ethereum.org/en/developers/docs/networking-layer/network-addresses/#enode) that other nodes can use to connect to it and exchange peer information. Make sure to replace the displayed IP address information (most probably `[::]`) with your externally accessible IP to get the actual `enode` URL. diff --git a/accounts/abi/abi.go b/accounts/abi/abi.go index cd2f4d7978bd1..81bbee2f2b4a6 100644 --- a/accounts/abi/abi.go +++ b/accounts/abi/abi.go @@ -95,7 +95,7 @@ func (abi ABI) getArguments(name string, data []byte) (Arguments, error) { args = event.Inputs } if args == nil { - return nil, errors.New("abi: could not locate named method or event") + return nil, fmt.Errorf("abi: could not locate named method or event: %s", name) } return args, nil } @@ -164,7 +164,7 @@ func (abi *ABI) UnmarshalJSON(data []byte) error { case "constructor": abi.Constructor = NewMethod("", "", Constructor, field.StateMutability, field.Constant, field.Payable, field.Inputs, nil) case "function": - name := overloadedName(field.Name, func(s string) bool { _, ok := abi.Methods[s]; return ok }) + name := ResolveNameConflict(field.Name, func(s string) bool { _, ok := abi.Methods[s]; return ok }) abi.Methods[name] = NewMethod(name, field.Name, Function, field.StateMutability, field.Constant, field.Payable, field.Inputs, field.Outputs) case "fallback": // New introduced function type in v0.6.0, check more detail @@ -184,9 +184,11 @@ func (abi *ABI) UnmarshalJSON(data []byte) error { } abi.Receive = NewMethod("", "", Receive, field.StateMutability, field.Constant, field.Payable, nil, nil) case "event": - name := overloadedName(field.Name, func(s string) bool { _, ok := abi.Events[s]; return ok }) + name := ResolveNameConflict(field.Name, func(s string) bool { _, ok := abi.Events[s]; return ok }) abi.Events[name] = NewEvent(name, field.Name, field.Anonymous, field.Inputs) case "error": + // Errors cannot be overloaded or overridden but are inherited, + // no need to resolve the name conflict here. abi.Errors[field.Name] = NewError(field.Name, field.Inputs) default: return fmt.Errorf("abi: could not recognize type %v of field %v", field.Type, field.Name) @@ -251,20 +253,3 @@ func UnpackRevert(data []byte) (string, error) { } return unpacked[0].(string), nil } - -// overloadedName returns the next available name for a given thing. -// Needed since solidity allows for overloading. -// -// e.g. if the abi contains Methods send, send1 -// overloadedName would return send2 for input send. -// -// overloadedName works for methods, events and errors. -func overloadedName(rawName string, isAvail func(string) bool) string { - name := rawName - ok := isAvail(name) - for idx := 0; ok; idx++ { - name = fmt.Sprintf("%s%d", rawName, idx) - ok = isAvail(name) - } - return name -} diff --git a/accounts/abi/abi_test.go b/accounts/abi/abi_test.go index cc8dfc61c389f..cd9433ca7f73a 100644 --- a/accounts/abi/abi_test.go +++ b/accounts/abi/abi_test.go @@ -1038,9 +1038,7 @@ func TestABI_EventById(t *testing.T) { } if event == nil { t.Errorf("We should find a event for topic %s, test #%d", topicID.Hex(), testnum) - } - - if event.ID != topicID { + } else if event.ID != topicID { t.Errorf("Event id %s does not match topic %s, test #%d", event.ID.Hex(), topicID.Hex(), testnum) } diff --git a/accounts/abi/argument.go b/accounts/abi/argument.go index c5326d5700a65..ed204e0a81dd7 100644 --- a/accounts/abi/argument.go +++ b/accounts/abi/argument.go @@ -18,6 +18,7 @@ package abi import ( "encoding/json" + "errors" "fmt" "reflect" "strings" @@ -79,7 +80,7 @@ func (arguments Arguments) isTuple() bool { func (arguments Arguments) Unpack(data []byte) ([]interface{}, error) { if len(data) == 0 { if len(arguments.NonIndexed()) != 0 { - return nil, fmt.Errorf("abi: attempting to unmarshall an empty string while arguments are expected") + return nil, errors.New("abi: attempting to unmarshall an empty string while arguments are expected") } return make([]interface{}, 0), nil } @@ -90,11 +91,11 @@ func (arguments Arguments) Unpack(data []byte) ([]interface{}, error) { func (arguments Arguments) UnpackIntoMap(v map[string]interface{}, data []byte) error { // Make sure map is not nil if v == nil { - return fmt.Errorf("abi: cannot unpack into a nil map") + return errors.New("abi: cannot unpack into a nil map") } if len(data) == 0 { if len(arguments.NonIndexed()) != 0 { - return fmt.Errorf("abi: attempting to unmarshall an empty string while arguments are expected") + return errors.New("abi: attempting to unmarshall an empty string while arguments are expected") } return nil // Nothing to unmarshal, return } @@ -116,7 +117,7 @@ func (arguments Arguments) Copy(v interface{}, values []interface{}) error { } if len(values) == 0 { if len(arguments.NonIndexed()) != 0 { - return fmt.Errorf("abi: attempting to copy no values while arguments are expected") + return errors.New("abi: attempting to copy no values while arguments are expected") } return nil // Nothing to copy, return } diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go index 31ab29be43ab0..74ceca5d0a1f7 100644 --- a/accounts/abi/bind/backends/simulated.go +++ b/accounts/abi/bind/backends/simulated.go @@ -63,11 +63,13 @@ type SimulatedBackend struct { database ethdb.Database // In memory database to store our testing data blockchain *core.BlockChain // Ethereum blockchain to handle the consensus - mu sync.Mutex - pendingBlock *types.Block // Currently pending block that will be imported on request - pendingState *state.StateDB // Currently pending state that will be the active on request + mu sync.Mutex + pendingBlock *types.Block // Currently pending block that will be imported on request + pendingState *state.StateDB // Currently pending state that will be the active on request + pendingReceipts types.Receipts // Currently receipts for the pending block - events *filters.EventSystem // Event system for filtering log events live + events *filters.EventSystem // for filtering log events live + filterSystem *filters.FilterSystem // for filtering database logs config *params.ChainConfig } @@ -84,8 +86,12 @@ func NewSimulatedBackendWithDatabase(database ethdb.Database, alloc core.Genesis database: database, blockchain: blockchain, config: genesis.Config, - events: filters.NewEventSystem(&filterBackend{database, blockchain}, false), } + + filterBackend := &filterBackend{database, blockchain, backend} + backend.filterSystem = filters.NewFilterSystem(filterBackend, filters.Config{}) + backend.events = filters.NewEventSystem(backend.filterSystem, false) + backend.rollback(blockchain.CurrentBlock()) return backend } @@ -105,16 +111,20 @@ func (b *SimulatedBackend) Close() error { // Commit imports all the pending transactions as a single block and starts a // fresh new state. -func (b *SimulatedBackend) Commit() { +func (b *SimulatedBackend) Commit() common.Hash { b.mu.Lock() defer b.mu.Unlock() if _, err := b.blockchain.InsertChain([]*types.Block{b.pendingBlock}); err != nil { panic(err) // This cannot happen unless the simulator is wrong, fail in that case } + blockHash := b.pendingBlock.Hash() + // Using the last inserted block here makes it possible to build on a side // chain after a fork. b.rollback(b.pendingBlock) + + return blockHash } // Rollback aborts all pending transactions, reverting to the last committed state. @@ -604,7 +614,7 @@ func (b *SimulatedBackend) callContract(ctx context.Context, call ethereum.CallM // User specified the legacy gas field, convert to 1559 gas typing call.GasFeeCap, call.GasTipCap = call.GasPrice, call.GasPrice } else { - // User specified 1559 gas feilds (or none), use those + // User specified 1559 gas fields (or none), use those if call.GasFeeCap == nil { call.GasFeeCap = new(big.Int) } @@ -662,7 +672,7 @@ func (b *SimulatedBackend) SendTransaction(ctx context.Context, tx *types.Transa return fmt.Errorf("invalid transaction nonce: got %d, want %d", tx.Nonce(), nonce) } // Include tx in chain - blocks, _ := core.GenerateChain(b.config, block, ethash.NewFaker(), b.database, 1, func(number int, block *core.BlockGen) { + blocks, receipts := core.GenerateChain(b.config, block, ethash.NewFaker(), b.database, 1, func(number int, block *core.BlockGen) { for _, tx := range b.pendingBlock.Transactions() { block.AddTxWithChain(b.blockchain, tx) } @@ -672,6 +682,7 @@ func (b *SimulatedBackend) SendTransaction(ctx context.Context, tx *types.Transa b.pendingBlock = blocks[0] b.pendingState, _ = state.New(b.pendingBlock.Root(), stateDB.Database(), nil) + b.pendingReceipts = receipts[0] return nil } @@ -683,7 +694,7 @@ func (b *SimulatedBackend) FilterLogs(ctx context.Context, query ethereum.Filter var filter *filters.Filter if query.BlockHash != nil { // Block filter requested, construct a single-shot filter - filter = filters.NewBlockFilter(&filterBackend{b.database, b.blockchain}, *query.BlockHash, query.Addresses, query.Topics) + filter = b.filterSystem.NewBlockFilter(*query.BlockHash, query.Addresses, query.Topics) } else { // Initialize unset filter boundaries to run from genesis to chain head from := int64(0) @@ -695,7 +706,7 @@ func (b *SimulatedBackend) FilterLogs(ctx context.Context, query ethereum.Filter to = query.ToBlock.Int64() } // Construct the range filter - filter = filters.NewRangeFilter(&filterBackend{b.database, b.blockchain}, from, to, query.Addresses, query.Topics) + filter = b.filterSystem.NewRangeFilter(from, to, query.Addresses, query.Topics) } // Run the filter and return all the logs logs, err := filter.Logs(ctx) @@ -816,11 +827,13 @@ func (m callMsg) AccessList() types.AccessList { return m.CallMsg.AccessList } // filterBackend implements filters.Backend to support filtering for logs without // taking bloom-bits acceleration structures into account. type filterBackend struct { - db ethdb.Database - bc *core.BlockChain + db ethdb.Database + bc *core.BlockChain + backend *SimulatedBackend } -func (fb *filterBackend) ChainDb() ethdb.Database { return fb.db } +func (fb *filterBackend) ChainDb() ethdb.Database { return fb.db } + func (fb *filterBackend) EventMux() *event.TypeMux { panic("not supported") } func (fb *filterBackend) HeaderByNumber(ctx context.Context, block rpc.BlockNumber) (*types.Header, error) { @@ -834,6 +847,10 @@ func (fb *filterBackend) HeaderByHash(ctx context.Context, hash common.Hash) (*t return fb.bc.GetHeaderByHash(hash), nil } +func (fb *filterBackend) PendingBlockAndReceipts() (*types.Block, types.Receipts) { + return fb.backend.pendingBlock, fb.backend.pendingReceipts +} + func (fb *filterBackend) GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) { number := rawdb.ReadHeaderNumber(fb.db, hash) if number == nil { @@ -842,19 +859,8 @@ func (fb *filterBackend) GetReceipts(ctx context.Context, hash common.Hash) (typ return rawdb.ReadReceipts(fb.db, hash, *number, fb.bc.Config()), nil } -func (fb *filterBackend) GetLogs(ctx context.Context, hash common.Hash) ([][]*types.Log, error) { - number := rawdb.ReadHeaderNumber(fb.db, hash) - if number == nil { - return nil, nil - } - receipts := rawdb.ReadReceipts(fb.db, hash, *number, fb.bc.Config()) - if receipts == nil { - return nil, nil - } - logs := make([][]*types.Log, len(receipts)) - for i, receipt := range receipts { - logs[i] = receipt.Logs - } +func (fb *filterBackend) GetLogs(ctx context.Context, hash common.Hash, number uint64) ([][]*types.Log, error) { + logs := rawdb.ReadLogs(fb.db, hash, number, fb.bc.Config()) return logs, nil } diff --git a/accounts/abi/bind/backends/simulated_test.go b/accounts/abi/bind/backends/simulated_test.go index 8a0cbe335778b..83367f0985238 100644 --- a/accounts/abi/bind/backends/simulated_test.go +++ b/accounts/abi/bind/backends/simulated_test.go @@ -655,8 +655,7 @@ func TestHeaderByNumber(t *testing.T) { } if latestBlockHeader == nil { t.Errorf("received a nil block header") - } - if latestBlockHeader.Number.Uint64() != uint64(0) { + } else if latestBlockHeader.Number.Uint64() != uint64(0) { t.Errorf("expected block header number 0, instead got %v", latestBlockHeader.Number.Uint64()) } @@ -1336,3 +1335,42 @@ func TestForkResendTx(t *testing.T) { t.Errorf("TX included in wrong block: %d", h) } } + +func TestCommitReturnValue(t *testing.T) { + testAddr := crypto.PubkeyToAddress(testKey.PublicKey) + sim := simTestBackend(testAddr) + defer sim.Close() + + startBlockHeight := sim.blockchain.CurrentBlock().NumberU64() + + // Test if Commit returns the correct block hash + h1 := sim.Commit() + if h1 != sim.blockchain.CurrentBlock().Hash() { + t.Error("Commit did not return the hash of the last block.") + } + + // Create a block in the original chain (containing a transaction to force different block hashes) + head, _ := sim.HeaderByNumber(context.Background(), nil) // Should be child's, good enough + gasPrice := new(big.Int).Add(head.BaseFee, big.NewInt(1)) + _tx := types.NewTransaction(0, testAddr, big.NewInt(1000), params.TxGas, gasPrice, nil) + tx, _ := types.SignTx(_tx, types.HomesteadSigner{}, testKey) + sim.SendTransaction(context.Background(), tx) + h2 := sim.Commit() + + // Create another block in the original chain + sim.Commit() + + // Fork at the first bock + if err := sim.Fork(context.Background(), h1); err != nil { + t.Errorf("forking: %v", err) + } + + // Test if Commit returns the correct block hash after the reorg + h2fork := sim.Commit() + if h2 == h2fork { + t.Error("The block in the fork and the original block are the same block!") + } + if sim.blockchain.GetHeader(h2fork, startBlockHeight+2) == nil { + t.Error("Could not retrieve the just created block (side-chain)") + } +} diff --git a/accounts/abi/bind/base_test.go b/accounts/abi/bind/base_test.go index 25b2f8a865f26..2307b9874b18f 100644 --- a/accounts/abi/bind/base_test.go +++ b/accounts/abi/bind/base_test.go @@ -115,7 +115,6 @@ func (mc *mockPendingCaller) PendingCallContract(ctx context.Context, call ether } func TestPassingBlockNumber(t *testing.T) { - mc := &mockPendingCaller{ mockCaller: &mockCaller{ codeAtBytes: []byte{1, 2, 3}, diff --git a/accounts/abi/bind/bind.go b/accounts/abi/bind/bind.go index 2bd8b6dde07f0..dac43f70e234b 100644 --- a/accounts/abi/bind/bind.go +++ b/accounts/abi/bind/bind.go @@ -43,6 +43,43 @@ const ( LangObjC ) +func isKeyWord(arg string) bool { + switch arg { + case "break": + case "case": + case "chan": + case "const": + case "continue": + case "default": + case "defer": + case "else": + case "fallthrough": + case "for": + case "func": + case "go": + case "goto": + case "if": + case "import": + case "interface": + case "iota": + case "map": + case "make": + case "new": + case "package": + case "range": + case "return": + case "select": + case "struct": + case "switch": + case "type": + case "var": + default: + return false + } + + return true +} + // Bind generates a Go wrapper around a contract ABI. This wrapper isn't meant // to be used as is in client code, but rather as an intermediate struct which // enforces compile time type safety and naming convention opposed to having to @@ -99,6 +136,7 @@ func Bind(types []string, abis []string, bytecodes []string, fsigs []map[string] // Normalize the method for capital cases and non-anonymous inputs/outputs normalized := original normalizedName := methodNormalizer[lang](alias(aliases, original.Name)) + // Ensure there is no duplicated identifier var identifiers = callIdentifiers if !original.IsConstant() { @@ -108,11 +146,12 @@ func Bind(types []string, abis []string, bytecodes []string, fsigs []map[string] return "", fmt.Errorf("duplicated identifier \"%s\"(normalized \"%s\"), use --alias for renaming", original.Name, normalizedName) } identifiers[normalizedName] = true + normalized.Name = normalizedName normalized.Inputs = make([]abi.Argument, len(original.Inputs)) copy(normalized.Inputs, original.Inputs) for j, input := range normalized.Inputs { - if input.Name == "" { + if input.Name == "" || isKeyWord(input.Name) { normalized.Inputs[j].Name = fmt.Sprintf("arg%d", j) } if hasStruct(input.Type) { @@ -152,12 +191,22 @@ func Bind(types []string, abis []string, bytecodes []string, fsigs []map[string] eventIdentifiers[normalizedName] = true normalized.Name = normalizedName + used := make(map[string]bool) normalized.Inputs = make([]abi.Argument, len(original.Inputs)) copy(normalized.Inputs, original.Inputs) for j, input := range normalized.Inputs { - if input.Name == "" { + if input.Name == "" || isKeyWord(input.Name) { normalized.Inputs[j].Name = fmt.Sprintf("arg%d", j) } + // Event is a bit special, we need to define event struct in binding, + // ensure there is no camel-case-style name conflict. + for index := 0; ; index++ { + if !used[capitalise(normalized.Inputs[j].Name)] { + used[capitalise(normalized.Inputs[j].Name)] = true + break + } + normalized.Inputs[j].Name = fmt.Sprintf("%s%d", normalized.Inputs[j].Name, index) + } if hasStruct(input.Type) { bindStructType[lang](input.Type, structs) } @@ -432,15 +481,22 @@ func bindStructTypeGo(kind abi.Type, structs map[string]*tmplStruct) string { if s, exist := structs[id]; exist { return s.Name } - var fields []*tmplField + var ( + names = make(map[string]bool) + fields []*tmplField + ) for i, elem := range kind.TupleElems { - field := bindStructTypeGo(*elem, structs) - fields = append(fields, &tmplField{Type: field, Name: capitalise(kind.TupleRawNames[i]), SolKind: *elem}) + name := capitalise(kind.TupleRawNames[i]) + name = abi.ResolveNameConflict(name, func(s string) bool { return names[s] }) + names[name] = true + fields = append(fields, &tmplField{Type: bindStructTypeGo(*elem, structs), Name: name, SolKind: *elem}) } name := kind.TupleRawName if name == "" { name = fmt.Sprintf("Struct%d", len(structs)) } + name = capitalise(name) + structs[id] = &tmplStruct{ Name: name, Fields: fields, diff --git a/accounts/abi/bind/bind_test.go b/accounts/abi/bind/bind_test.go index 87e8187f09ca0..5fa803849df6f 100644 --- a/accounts/abi/bind/bind_test.go +++ b/accounts/abi/bind/bind_test.go @@ -1954,6 +1954,91 @@ var bindTests = []struct { } `, }, + { + name: `NameConflict`, + contract: ` + // SPDX-License-Identifier: GPL-3.0 + pragma solidity >=0.4.22 <0.9.0; + contract oracle { + struct request { + bytes data; + bytes _data; + } + event log (int msg, int _msg); + function addRequest(request memory req) public pure {} + function getRequest() pure public returns (request memory) { + return request("", ""); + } + } + `, + bytecode: []string{"0x608060405234801561001057600080fd5b5061042b806100206000396000f3fe608060405234801561001057600080fd5b50600436106100365760003560e01c8063c2bb515f1461003b578063cce7b04814610059575b600080fd5b610043610075565b60405161005091906101af565b60405180910390f35b610073600480360381019061006e91906103ac565b6100b5565b005b61007d6100b8565b604051806040016040528060405180602001604052806000815250815260200160405180602001604052806000815250815250905090565b50565b604051806040016040528060608152602001606081525090565b600081519050919050565b600082825260208201905092915050565b60005b8381101561010c5780820151818401526020810190506100f1565b8381111561011b576000848401525b50505050565b6000601f19601f8301169050919050565b600061013d826100d2565b61014781856100dd565b93506101578185602086016100ee565b61016081610121565b840191505092915050565b600060408301600083015184820360008601526101888282610132565b915050602083015184820360208601526101a28282610132565b9150508091505092915050565b600060208201905081810360008301526101c9818461016b565b905092915050565b6000604051905090565b600080fd5b600080fd5b600080fd5b7f4e487b7100000000000000000000000000000000000000000000000000000000600052604160045260246000fd5b61022282610121565b810181811067ffffffffffffffff82111715610241576102406101ea565b5b80604052505050565b60006102546101d1565b90506102608282610219565b919050565b600080fd5b600080fd5b600080fd5b600067ffffffffffffffff82111561028f5761028e6101ea565b5b61029882610121565b9050602081019050919050565b82818337600083830152505050565b60006102c76102c284610274565b61024a565b9050828152602081018484840111156102e3576102e261026f565b5b6102ee8482856102a5565b509392505050565b600082601f83011261030b5761030a61026a565b5b813561031b8482602086016102b4565b91505092915050565b60006040828403121561033a576103396101e5565b5b610344604061024a565b9050600082013567ffffffffffffffff81111561036457610363610265565b5b610370848285016102f6565b600083015250602082013567ffffffffffffffff81111561039457610393610265565b5b6103a0848285016102f6565b60208301525092915050565b6000602082840312156103c2576103c16101db565b5b600082013567ffffffffffffffff8111156103e0576103df6101e0565b5b6103ec84828501610324565b9150509291505056fea264697066735822122033bca1606af9b6aeba1673f98c52003cec19338539fb44b86690ce82c51483b564736f6c634300080e0033"}, + abi: []string{`[ { "anonymous": false, "inputs": [ { "indexed": false, "internalType": "int256", "name": "msg", "type": "int256" }, { "indexed": false, "internalType": "int256", "name": "_msg", "type": "int256" } ], "name": "log", "type": "event" }, { "inputs": [ { "components": [ { "internalType": "bytes", "name": "data", "type": "bytes" }, { "internalType": "bytes", "name": "_data", "type": "bytes" } ], "internalType": "struct oracle.request", "name": "req", "type": "tuple" } ], "name": "addRequest", "outputs": [], "stateMutability": "pure", "type": "function" }, { "inputs": [], "name": "getRequest", "outputs": [ { "components": [ { "internalType": "bytes", "name": "data", "type": "bytes" }, { "internalType": "bytes", "name": "_data", "type": "bytes" } ], "internalType": "struct oracle.request", "name": "", "type": "tuple" } ], "stateMutability": "pure", "type": "function" } ]`}, + imports: ` + "math/big" + + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/eth/ethconfig" + `, + tester: ` + var ( + key, _ = crypto.GenerateKey() + user, _ = bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) + sim = backends.NewSimulatedBackend(core.GenesisAlloc{user.From: {Balance: big.NewInt(1000000000000000000)}}, ethconfig.Defaults.Miner.GasCeil) + ) + defer sim.Close() + + _, tx, _, err := DeployNameConflict(user, sim) + if err != nil { + t.Fatalf("DeployNameConflict() got err %v; want nil err", err) + } + sim.Commit() + + if _, err = bind.WaitDeployed(nil, sim, tx); err != nil { + t.Logf("Deployment tx: %+v", tx) + t.Errorf("bind.WaitDeployed(nil, %T, ) got err %v; want nil err", sim, err) + } + `, + }, + { + name: "RangeKeyword", + contract: ` + // SPDX-License-Identifier: GPL-3.0 + pragma solidity >=0.4.22 <0.9.0; + contract keywordcontract { + function functionWithKeywordParameter(range uint256) public pure {} + } + `, + bytecode: []string{"0x608060405234801561001057600080fd5b5060dc8061001f6000396000f3fe6080604052348015600f57600080fd5b506004361060285760003560e01c8063527a119f14602d575b600080fd5b60436004803603810190603f9190605b565b6045565b005b50565b6000813590506055816092565b92915050565b600060208284031215606e57606d608d565b5b6000607a848285016048565b91505092915050565b6000819050919050565b600080fd5b6099816083565b811460a357600080fd5b5056fea2646970667358221220d4f4525e2615516394055d369fb17df41c359e5e962734f27fd683ea81fd9db164736f6c63430008070033"}, + abi: []string{`[{"inputs":[{"internalType":"uint256","name":"range","type":"uint256"}],"name":"functionWithKeywordParameter","outputs":[],"stateMutability":"pure","type":"function"}]`}, + imports: ` + "math/big" + + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/eth/ethconfig" + `, + tester: ` + var ( + key, _ = crypto.GenerateKey() + user, _ = bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) + sim = backends.NewSimulatedBackend(core.GenesisAlloc{user.From: {Balance: big.NewInt(1000000000000000000)}}, ethconfig.Defaults.Miner.GasCeil) + ) + _, tx, _, err := DeployRangeKeyword(user, sim) + if err != nil { + t.Fatalf("error deploying contract: %v", err) + } + sim.Commit() + + if _, err = bind.WaitDeployed(nil, sim, tx); err != nil { + t.Errorf("error deploying the contract: %v", err) + } + `, + }, } // Tests that packages generated by the binder can be successfully compiled and diff --git a/accounts/abi/error.go b/accounts/abi/error.go index e564c10c2f840..f53c996def14d 100644 --- a/accounts/abi/error.go +++ b/accounts/abi/error.go @@ -30,11 +30,13 @@ type Error struct { Name string Inputs Arguments str string + // Sig contains the string signature according to the ABI spec. - // e.g. event foo(uint32 a, int b) = "foo(uint32,int256)" + // e.g. error foo(uint32 a, int b) = "foo(uint32,int256)" // Please note that "int" is substitute for its canonical representation "int256" Sig string - // ID returns the canonical representation of the event's signature used by the + + // ID returns the canonical representation of the error's signature used by the // abi definition to identify event names and types. ID common.Hash } diff --git a/accounts/abi/error_handling.go b/accounts/abi/error_handling.go index f0f71b6c91646..7add7072925e6 100644 --- a/accounts/abi/error_handling.go +++ b/accounts/abi/error_handling.go @@ -73,7 +73,6 @@ func typeCheck(t Type, value reflect.Value) error { } else { return nil } - } // typeErr returns a formatted type casting error. diff --git a/accounts/abi/event.go b/accounts/abi/event.go index b238a36d7ceab..f9457b86afebf 100644 --- a/accounts/abi/event.go +++ b/accounts/abi/event.go @@ -29,24 +29,27 @@ import ( // don't get the signature canonical representation as the first LOG topic. type Event struct { // Name is the event name used for internal representation. It's derived from - // the raw name and a suffix will be added in the case of a event overload. + // the raw name and a suffix will be added in the case of event overloading. // // e.g. // These are two events that have the same name: // * foo(int,int) // * foo(uint,uint) - // The event name of the first one wll be resolved as foo while the second one + // The event name of the first one will be resolved as foo while the second one // will be resolved as foo0. Name string + // RawName is the raw event name parsed from ABI. RawName string Anonymous bool Inputs Arguments str string + // Sig contains the string signature according to the ABI spec. // e.g. event foo(uint32 a, int b) = "foo(uint32,int256)" // Please note that "int" is substitute for its canonical representation "int256" Sig string + // ID returns the canonical representation of the event's signature used by the // abi definition to identify event names and types. ID common.Hash diff --git a/accounts/abi/event_test.go b/accounts/abi/event_test.go index 3332f8a07216c..8f73419496ba5 100644 --- a/accounts/abi/event_test.go +++ b/accounts/abi/event_test.go @@ -161,7 +161,6 @@ func TestEventMultiValueWithArrayUnpack(t *testing.T) { } func TestEventTupleUnpack(t *testing.T) { - type EventTransfer struct { Value *big.Int } diff --git a/accounts/abi/reflect.go b/accounts/abi/reflect.go index 35e5556d2c5a7..7917fa98095bd 100644 --- a/accounts/abi/reflect.go +++ b/accounts/abi/reflect.go @@ -99,7 +99,7 @@ func mustArrayToByteSlice(value reflect.Value) reflect.Value { func set(dst, src reflect.Value) error { dstType, srcType := dst.Type(), src.Type() switch { - case dstType.Kind() == reflect.Interface && dst.Elem().IsValid(): + case dstType.Kind() == reflect.Interface && dst.Elem().IsValid() && (dst.Elem().Type().Kind() == reflect.Ptr || dst.Elem().CanSet()): return set(dst.Elem(), src) case dstType.Kind() == reflect.Ptr && dstType.Elem() != reflect.TypeOf(big.Int{}): return set(dst.Elem(), src) @@ -220,7 +220,6 @@ func mapArgNamesToStructFields(argNames []string, value reflect.Value) (map[stri // second round ~~~ for _, argName := range argNames { - structFieldName := ToCamelCase(argName) if structFieldName == "" { diff --git a/accounts/abi/reflect_test.go b/accounts/abi/reflect_test.go index cf13a79da84e6..76ef1ad2aa39b 100644 --- a/accounts/abi/reflect_test.go +++ b/accounts/abi/reflect_test.go @@ -32,7 +32,7 @@ type reflectTest struct { var reflectTests = []reflectTest{ { - name: "OneToOneCorrespondance", + name: "OneToOneCorrespondence", args: []string{"fieldA"}, struc: struct { FieldA int `abi:"fieldA"` diff --git a/accounts/abi/selector_parser.go b/accounts/abi/selector_parser.go index 88114e288eb39..d5472e374f5dd 100644 --- a/accounts/abi/selector_parser.go +++ b/accounts/abi/selector_parser.go @@ -166,7 +166,7 @@ func ParseSelector(unescapedSelector string) (SelectorMarshaling, error) { return SelectorMarshaling{}, fmt.Errorf("failed to parse selector '%s': unexpected string '%s'", unescapedSelector, rest) } - // Reassemble the fake ABI and constuct the JSON + // Reassemble the fake ABI and construct the JSON fakeArgs, err := assembleArgs(args) if err != nil { return SelectorMarshaling{}, fmt.Errorf("failed to parse selector: %v", err) diff --git a/accounts/abi/type.go b/accounts/abi/type.go index fd75f586a99db..008b665b1aeeb 100644 --- a/accounts/abi/type.go +++ b/accounts/abi/type.go @@ -163,22 +163,26 @@ func NewType(t string, internalType string, components []ArgumentMarshaling) (ty elems []*Type names []string expression string // canonical parameter expression + used = make(map[string]bool) ) expression += "(" - overloadedNames := make(map[string]string) for idx, c := range components { cType, err := NewType(c.Type, c.InternalType, c.Components) if err != nil { return Type{}, err } - fieldName, err := overloadedArgName(c.Name, overloadedNames) + name := ToCamelCase(c.Name) + if name == "" { + return Type{}, errors.New("abi: purely anonymous or underscored field is not supported") + } + fieldName := ResolveNameConflict(name, func(s string) bool { return used[s] }) if err != nil { return Type{}, err } + used[fieldName] = true if !isValidFieldName(fieldName) { return Type{}, fmt.Errorf("field %d has invalid name", idx) } - overloadedNames[fieldName] = fieldName fields = append(fields, reflect.StructField{ Name: fieldName, // reflect.StructOf will panic for any exported field. Type: cType.GetType(), @@ -255,20 +259,6 @@ func (t Type) GetType() reflect.Type { } } -func overloadedArgName(rawName string, names map[string]string) (string, error) { - fieldName := ToCamelCase(rawName) - if fieldName == "" { - return "", errors.New("abi: purely anonymous or underscored field is not supported") - } - // Handle overloaded fieldNames - _, ok := names[fieldName] - for idx := 0; ok; idx++ { - fieldName = fmt.Sprintf("%s%d", ToCamelCase(rawName), idx) - _, ok = names[fieldName] - } - return fieldName, nil -} - // String implements Stringer. func (t Type) String() (out string) { return t.stringKind diff --git a/accounts/abi/unpack.go b/accounts/abi/unpack.go index 43cd6c64575c7..800789295c19a 100644 --- a/accounts/abi/unpack.go +++ b/accounts/abi/unpack.go @@ -115,7 +115,6 @@ func ReadFixedBytes(t Type, word []byte) (interface{}, error) { reflect.Copy(array, reflect.ValueOf(word[0:t.Size])) return array.Interface(), nil - } // forEachUnpack iteratively unpack elements. @@ -255,7 +254,7 @@ func toGoType(index int, t Type, output []byte) (interface{}, error) { // lengthPrefixPointsTo interprets a 32 byte slice as an offset and then determines which indices to look to decode the type. func lengthPrefixPointsTo(index int, output []byte) (start int, length int, err error) { - bigOffsetEnd := big.NewInt(0).SetBytes(output[index : index+32]) + bigOffsetEnd := new(big.Int).SetBytes(output[index : index+32]) bigOffsetEnd.Add(bigOffsetEnd, common.Big32) outputLength := big.NewInt(int64(len(output))) @@ -268,11 +267,9 @@ func lengthPrefixPointsTo(index int, output []byte) (start int, length int, err } offsetEnd := int(bigOffsetEnd.Uint64()) - lengthBig := big.NewInt(0).SetBytes(output[offsetEnd-32 : offsetEnd]) + lengthBig := new(big.Int).SetBytes(output[offsetEnd-32 : offsetEnd]) - totalSize := big.NewInt(0) - totalSize.Add(totalSize, bigOffsetEnd) - totalSize.Add(totalSize, lengthBig) + totalSize := new(big.Int).Add(bigOffsetEnd, lengthBig) if totalSize.BitLen() > 63 { return 0, 0, fmt.Errorf("abi: length larger than int64: %v", totalSize) } @@ -287,7 +284,7 @@ func lengthPrefixPointsTo(index int, output []byte) (start int, length int, err // tuplePointsTo resolves the location reference for dynamic tuple. func tuplePointsTo(index int, output []byte) (start int, err error) { - offset := big.NewInt(0).SetBytes(output[index : index+32]) + offset := new(big.Int).SetBytes(output[index : index+32]) outputLen := big.NewInt(int64(len(output))) if offset.Cmp(outputLen) > 0 { diff --git a/accounts/abi/unpack_test.go b/accounts/abi/unpack_test.go index bf40c301b5f7c..363e0cd5943ec 100644 --- a/accounts/abi/unpack_test.go +++ b/accounts/abi/unpack_test.go @@ -352,6 +352,11 @@ func TestMethodMultiReturn(t *testing.T) { &[]interface{}{&expected.Int, &expected.String}, "", "Can unpack into a slice", + }, { + &[]interface{}{&bigint, ""}, + &[]interface{}{&expected.Int, expected.String}, + "", + "Can unpack into a slice without indirection", }, { &[2]interface{}{&bigint, new(string)}, &[2]interface{}{&expected.Int, &expected.String}, @@ -424,7 +429,7 @@ func TestMultiReturnWithStringArray(t *testing.T) { } buff := new(bytes.Buffer) buff.Write(common.Hex2Bytes("000000000000000000000000000000000000000000000000000000005c1b78ea0000000000000000000000000000000000000000000000000000000000000006000000000000000000000000000000000000000000000001a055690d9db80000000000000000000000000000ab1257528b3782fb40d7ed5f72e624b744dffb2f00000000000000000000000000000000000000000000000000000000000000c00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000008457468657265756d000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001048656c6c6f2c20457468657265756d2100000000000000000000000000000000")) - temp, _ := big.NewInt(0).SetString("30000000000000000000", 10) + temp, _ := new(big.Int).SetString("30000000000000000000", 10) ret1, ret1Exp := new([3]*big.Int), [3]*big.Int{big.NewInt(1545304298), big.NewInt(6), temp} ret2, ret2Exp := new(common.Address), common.HexToAddress("ab1257528b3782fb40d7ed5f72e624b744dffb2f") ret3, ret3Exp := new([2]string), [2]string{"Ethereum", "Hello, Ethereum!"} diff --git a/accounts/abi/utils.go b/accounts/abi/utils.go new file mode 100644 index 0000000000000..e24df5b7003d8 --- /dev/null +++ b/accounts/abi/utils.go @@ -0,0 +1,41 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package abi + +import "fmt" + +// ResolveNameConflict returns the next available name for a given thing. +// This helper can be used for lots of purposes: +// +// - In solidity function overloading is supported, this function can fix +// the name conflicts of overloaded functions. +// - In golang binding generation, the parameter(in function, event, error, +// and struct definition) name will be converted to camelcase style which +// may eventually lead to name conflicts. +// +// Name conflicts are mostly resolved by adding number suffix. +// e.g. if the abi contains Methods send, send1 +// ResolveNameConflict would return send2 for input send. +func ResolveNameConflict(rawName string, used func(string) bool) string { + name := rawName + ok := used(name) + for idx := 0; ok; idx++ { + name = fmt.Sprintf("%s%d", rawName, idx) + ok = used(name) + } + return name +} diff --git a/accounts/external/backend.go b/accounts/external/backend.go index e3f754eafcc42..d403b7e562d41 100644 --- a/accounts/external/backend.go +++ b/accounts/external/backend.go @@ -152,10 +152,6 @@ func (api *ExternalSigner) SelfDerive(bases []accounts.DerivationPath, chain eth log.Error("operation SelfDerive not supported on external signers") } -func (api *ExternalSigner) signHash(account accounts.Account, hash []byte) ([]byte, error) { - return []byte{}, fmt.Errorf("operation not supported on external signers") -} - // SignData signs keccak256(data). The mimetype parameter describes the type of data being signed func (api *ExternalSigner) SignData(account accounts.Account, mimeType string, data []byte) ([]byte, error) { var res hexutil.Bytes diff --git a/accounts/keystore/account_cache_test.go b/accounts/keystore/account_cache_test.go index fda0e5667c2aa..daea497d1ae76 100644 --- a/accounts/keystore/account_cache_test.go +++ b/accounts/keystore/account_cache_test.go @@ -318,7 +318,7 @@ func waitForAccounts(wantAccounts []accounts.Account, ks *KeyStore) error { func TestUpdatedKeyfileContents(t *testing.T) { t.Parallel() - // Create a temporary kesytore to test with + // Create a temporary keystore to test with rand.Seed(time.Now().UnixNano()) dir := filepath.Join(os.TempDir(), fmt.Sprintf("eth-keystore-updatedkeyfilecontents-test-%d-%d", os.Getpid(), rand.Int())) ks := NewKeyStore(dir, LightScryptN, LightScryptP) @@ -383,7 +383,7 @@ func TestUpdatedKeyfileContents(t *testing.T) { time.Sleep(1000 * time.Millisecond) // Now replace file contents with crap - if err := os.WriteFile(file, []byte("foo"), 0644); err != nil { + if err := os.WriteFile(file, []byte("foo"), 0600); err != nil { t.Fatal(err) return } diff --git a/accounts/keystore/file_cache.go b/accounts/keystore/file_cache.go index b3ecf8946b53b..79f9a29637431 100644 --- a/accounts/keystore/file_cache.go +++ b/accounts/keystore/file_cache.go @@ -39,7 +39,7 @@ type fileCache struct { func (fc *fileCache) scan(keyDir string) (mapset.Set, mapset.Set, mapset.Set, error) { t0 := time.Now() - // List all the failes from the keystore folder + // List all the files from the keystore folder files, err := os.ReadDir(keyDir) if err != nil { return nil, nil, nil, err @@ -61,7 +61,7 @@ func (fc *fileCache) scan(keyDir string) (mapset.Set, mapset.Set, mapset.Set, er log.Trace("Ignoring file on account scan", "path", path) continue } - // Gather the set of all and fresly modified files + // Gather the set of all and freshly modified files all.Add(path) info, err := fi.Info() diff --git a/accounts/keystore/keystore_test.go b/accounts/keystore/keystore_test.go index 80c4d643e8e18..4cdf0b1ed6ce8 100644 --- a/accounts/keystore/keystore_test.go +++ b/accounts/keystore/keystore_test.go @@ -214,7 +214,7 @@ func TestSignRace(t *testing.T) { // Tests that the wallet notifier loop starts and stops correctly based on the // addition and removal of wallet event subscriptions. func TestWalletNotifierLifecycle(t *testing.T) { - // Create a temporary kesytore to test with + // Create a temporary keystore to test with _, ks := tmpKeyStore(t, false) // Ensure that the notification updater is not running yet @@ -377,7 +377,6 @@ func TestImportExport(t *testing.T) { if _, err = ks2.Import(json, "new", "new"); err == nil { t.Errorf("importing a key twice succeeded") } - } // TestImportRace tests the keystore on races. @@ -402,7 +401,6 @@ func TestImportRace(t *testing.T) { if _, err := ks2.Import(json, "new", "new"); err != nil { atomic.AddUint32(&atom, 1) } - }() } wg.Wait() diff --git a/accounts/keystore/passphrase.go b/accounts/keystore/passphrase.go index 22772e93102f4..1701fbf53634e 100644 --- a/accounts/keystore/passphrase.go +++ b/accounts/keystore/passphrase.go @@ -138,7 +138,6 @@ func (ks keyStorePassphrase) JoinPath(filename string) string { // Encryptdata encrypts the data given as 'data' with the password 'auth'. func EncryptDataV3(data, auth []byte, scryptN, scryptP int) (CryptoJSON, error) { - salt := make([]byte, 32) if _, err := io.ReadFull(rand.Reader, salt); err != nil { panic("reading from crypto/rand failed: " + err.Error()) @@ -341,7 +340,6 @@ func getKDFKey(cryptoJSON CryptoJSON, auth string) ([]byte, error) { r := ensureInt(cryptoJSON.KDFParams["r"]) p := ensureInt(cryptoJSON.KDFParams["p"]) return scrypt.Key(authArray, salt, n, r, p, dkLen) - } else if cryptoJSON.KDF == "pbkdf2" { c := ensureInt(cryptoJSON.KDFParams["c"]) prf := cryptoJSON.KDFParams["prf"].(string) diff --git a/accounts/keystore/passphrase_test.go b/accounts/keystore/passphrase_test.go index b94fce8edcae7..1356b317806dc 100644 --- a/accounts/keystore/passphrase_test.go +++ b/accounts/keystore/passphrase_test.go @@ -52,7 +52,7 @@ func TestKeyEncryptDecrypt(t *testing.T) { t.Errorf("test %d: key address mismatch: have %x, want %x", i, key.Address, address) } // Recrypt with a new password and start over - password += "new data appended" + password += "new data appended" // nolint: gosec if keyjson, err = EncryptKey(key, password, veryLightScryptN, veryLightScryptP); err != nil { t.Errorf("test %d: failed to recrypt key %v", i, err) } diff --git a/accounts/scwallet/securechannel.go b/accounts/scwallet/securechannel.go index 10887a8b43d0f..b1b533eb72438 100644 --- a/accounts/scwallet/securechannel.go +++ b/accounts/scwallet/securechannel.go @@ -178,7 +178,7 @@ func (s *SecureChannelSession) mutuallyAuthenticate() error { return err } if response.Sw1 != 0x90 || response.Sw2 != 0x00 { - return fmt.Errorf("got unexpected response from MUTUALLY_AUTHENTICATE: 0x%x%x", response.Sw1, response.Sw2) + return fmt.Errorf("got unexpected response from MUTUALLY_AUTHENTICATE: %#x%x", response.Sw1, response.Sw2) } if len(response.Data) != scSecretLength { @@ -261,7 +261,7 @@ func (s *SecureChannelSession) transmitEncrypted(cla, ins, p1, p2 byte, data []b rapdu.deserialize(plainData) if rapdu.Sw1 != sw1Ok { - return nil, fmt.Errorf("unexpected response status Cla=0x%x, Ins=0x%x, Sw=0x%x%x", cla, ins, rapdu.Sw1, rapdu.Sw2) + return nil, fmt.Errorf("unexpected response status Cla=%#x, Ins=%#x, Sw=%#x%x", cla, ins, rapdu.Sw1, rapdu.Sw2) } return rapdu, nil diff --git a/accounts/scwallet/wallet.go b/accounts/scwallet/wallet.go index 2a2b83bd1b156..5082dec1cb33e 100644 --- a/accounts/scwallet/wallet.go +++ b/accounts/scwallet/wallet.go @@ -167,7 +167,7 @@ func transmit(card *pcsc.Card, command *commandAPDU) (*responseAPDU, error) { } if response.Sw1 != sw1Ok { - return nil, fmt.Errorf("unexpected insecure response status Cla=0x%x, Ins=0x%x, Sw=0x%x%x", command.Cla, command.Ins, response.Sw1, response.Sw2) + return nil, fmt.Errorf("unexpected insecure response status Cla=%#x, Ins=%#x, Sw=%#x%x", command.Cla, command.Ins, response.Sw1, response.Sw2) } return response, nil diff --git a/accounts/url_test.go b/accounts/url_test.go index bd6f35fa2a0ea..239aa06d227ba 100644 --- a/accounts/url_test.go +++ b/accounts/url_test.go @@ -32,9 +32,10 @@ func TestURLParsing(t *testing.T) { t.Errorf("expected: %v, got: %v", "ethereum.org", url.Path) } - _, err = parseURL("ethereum.org") - if err == nil { - t.Error("expected err, got: nil") + for _, u := range []string{"ethereum.org", ""} { + if _, err = parseURL(u); err == nil { + t.Errorf("input %v, expected err, got: nil", u) + } } } diff --git a/accounts/usbwallet/trezor.go b/accounts/usbwallet/trezor.go index c2182b88d03be..e385682a58334 100644 --- a/accounts/usbwallet/trezor.go +++ b/accounts/usbwallet/trezor.go @@ -196,10 +196,10 @@ func (w *trezorDriver) trezorDerive(derivationPath []uint32) (common.Address, er if _, err := w.trezorExchange(&trezor.EthereumGetAddress{AddressN: derivationPath}, address); err != nil { return common.Address{}, err } - if addr := address.GetAddressBin(); len(addr) > 0 { // Older firmwares use binary fomats + if addr := address.GetAddressBin(); len(addr) > 0 { // Older firmwares use binary formats return common.BytesToAddress(addr), nil } - if addr := address.GetAddressHex(); len(addr) > 0 { // Newer firmwares use hexadecimal fomats + if addr := address.GetAddressHex(); len(addr) > 0 { // Newer firmwares use hexadecimal formats return common.HexToAddress(addr), nil } return common.Address{}, errors.New("missing derived address") diff --git a/accounts/usbwallet/wallet.go b/accounts/usbwallet/wallet.go index 382f3ddaee217..0e399a6d09abb 100644 --- a/accounts/usbwallet/wallet.go +++ b/accounts/usbwallet/wallet.go @@ -380,7 +380,7 @@ func (w *wallet) selfDerive() { // of legacy-ledger, the first account on the legacy-path will // be shown to the user, even if we don't actively track it if i < len(nextAddrs)-1 { - w.log.Info("Skipping trakcking first account on legacy path, use personal.deriveAccount(,, false) to track", + w.log.Info("Skipping tracking first account on legacy path, use personal.deriveAccount(,, false) to track", "path", path, "address", nextAddrs[i]) break } @@ -526,7 +526,6 @@ func (w *wallet) signHash(account accounts.Account, hash []byte) ([]byte, error) // SignData signs keccak256(data). The mimetype parameter describes the type of data being signed func (w *wallet) SignData(account accounts.Account, mimeType string, data []byte) ([]byte, error) { - // Unless we are doing 712 signing, simply dispatch to signHash if !(mimeType == accounts.MimetypeTypedData && len(data) == 66 && data[0] == 0x19 && data[1] == 0x01) { return w.signHash(account, crypto.Keccak256(data)) diff --git a/build/checksums.txt b/build/checksums.txt index 4d6176ecbe59b..f7b13a0330f28 100644 --- a/build/checksums.txt +++ b/build/checksums.txt @@ -1,58 +1,38 @@ # This file contains sha256 checksums of optional build dependencies. -efd43e0f1402e083b73a03d444b7b6576bb4c539ac46208b63a916b69aca4088 go1.18.1.src.tar.gz -3703e9a0db1000f18c0c7b524f3d378aac71219b4715a6a4c5683eb639f41a4d go1.18.1.darwin-amd64.tar.gz -6d5641a06edba8cd6d425fb0adad06bad80e2afe0fa91b4aa0e5aed1bc78f58e go1.18.1.darwin-arm64.tar.gz -b9a9063d4265d8ccc046c9b314194d6eadc47e56d0d637db81e98e68aad45035 go1.18.1.freebsd-386.tar.gz -2bc1c138d645e37dbbc63517dd1cf1bf33fc4cb95f442a6384df0418b5134e9f go1.18.1.freebsd-amd64.tar.gz -9a8df5dde9058f08ac01ecfaae42534610db398e487138788c01da26a0d41ff9 go1.18.1.linux-386.tar.gz -b3b815f47ababac13810fc6021eb73d65478e0b2db4b09d348eefad9581a2334 go1.18.1.linux-amd64.tar.gz -56a91851c97fb4697077abbca38860f735c32b38993ff79b088dac46e4735633 go1.18.1.linux-arm64.tar.gz -9edc01c8e7db64e9ceeffc8258359e027812886ceca3444e83c4eb96ddb068ee go1.18.1.linux-armv6l.tar.gz -33db623d1eecf362fe365107c12efc90eff0b9609e0b3345e258388019cb552a go1.18.1.linux-ppc64le.tar.gz -5d9301324148ed4dbfaa0800da43a843ffd65c834ee73fcf087255697c925f74 go1.18.1.linux-s390x.tar.gz -49ae65551acbfaa57b52fbefa0350b2072512ae3103b8cf1a919a02626dbc743 go1.18.1.windows-386.zip -c30bc3f1f7314a953fe208bd9cd5e24bd9403392a6c556ced3677f9f70f71fe1 go1.18.1.windows-amd64.zip -2c4a8265030eac37f906634f5c13c22c3d0ea725f2488e1bca005c6b981653d7 go1.18.1.windows-arm64.zip +9920d3306a1ac536cdd2c796d6cb3c54bc559c226fc3cc39c32f1e0bd7f50d2a go1.18.5.src.tar.gz +828eeca8b5abea3e56921df8fa4b1101380a5ebcfee10acbc8ffe7ec0bf5876b go1.18.5.darwin-amd64.tar.gz +923a377c6fc9a2c789f5db61c24b8f64133f7889056897449891f256af34065f go1.18.5.darwin-arm64.tar.gz +c3d90264a706e2d88cfb44126dc6f0d008a48f00732e04bc377cea1a2b716a7c go1.18.5.freebsd-386.tar.gz +0de23843c568d388bc0f0e390a8966938cccaae0d74b698325f7175bac04e0c6 go1.18.5.freebsd-amd64.tar.gz +0c44f85d146c6f98c34e8ff436a42af22e90e36fe232d3d9d3101f23fd61362b go1.18.5.linux-386.tar.gz +9e5de37f9c49942c601b191ac5fba404b868bfc21d446d6960acc12283d6e5f2 go1.18.5.linux-amd64.tar.gz +006f6622718212363fa1ff004a6ab4d87bbbe772ec5631bab7cac10be346e4f1 go1.18.5.linux-arm64.tar.gz +d5ac34ac5f060a5274319aa04b7b11e41b123bd7887d64efb5f44ead236957af go1.18.5.linux-armv6l.tar.gz +2e37fb9c7cbaedd4e729492d658aa4cde821fc94117391a8105c13b25ca1c84b go1.18.5.linux-ppc64le.tar.gz +e3d536e7873639f85353e892444f83b14cb6670603961f215986ae8e28e8e07a go1.18.5.linux-s390x.tar.gz +7b3142ec0c5db991e7f73a231662a92429b90ee151fe47557acb566d8d9ae4d3 go1.18.5.windows-386.zip +73753620602d4b4469770040c53db55e5dd6af2ad07ecc18f71f164c3224eaad go1.18.5.windows-amd64.zip +4d154626affff12ef73ea1017af0e5b52dbc839ef92f6f9e76cf4f71278a5744 go1.18.5.windows-arm64.zip -03c181fc1bb29ea3e73cbb23399c43b081063833a7cf7554b94e5a98308df53e golangci-lint-1.45.2-linux-riscv64.deb -08a50bbbf451ede6d5354179eb3e14a5634e156dfa92cb9a2606f855a637e35b golangci-lint-1.45.2-linux-ppc64le.rpm -0d12f6ec1296b5a70e392aa88cd2295cceef266165eb7028e675f455515dd1c9 golangci-lint-1.45.2-linux-armv7.deb -10f2846e2e50e4ea8ae426ee62dcd2227b23adddd8e991aa3c065927ac948735 golangci-lint-1.45.2-linux-ppc64le.deb -1463049b744871168095e3e8f687247d6040eeb895955b869889ea151e0603ab golangci-lint-1.45.2-linux-arm64.tar.gz -15720f9c4c6f9324af695f081dc189adc7751b255759e78d7b2df1d7e9192533 golangci-lint-1.45.2-linux-amd64.deb -166d922e4d3cfe3d47786c590154a9c8ea689dff0aa92b73d2f5fc74fc570c29 golangci-lint-1.45.2-linux-arm64.rpm -1a3754c69f7cc19ab89cbdcc2550da4cf9abb3120383c6b3bd440c1ec22da2e6 golangci-lint-1.45.2-freebsd-386.tar.gz -1dec0aa46d4f0d241863b573f70129bdf1de9c595cf51172a840a588a4cd9fc5 golangci-lint-1.45.2-windows-amd64.zip -3198453806517c1ad988229f5e758ef850e671203f46d6905509df5bdf4dc24b golangci-lint-1.45.2-freebsd-armv7.tar.gz -46a3cd1749d7b98adc2dc01510ddbe21abe42689c8a53fb0e81662713629f215 golangci-lint-1.45.2-linux-386.deb -4e28bfb593d464b9e160f2acd5b71993836a183270bf8299b78ad31f7a168c0d golangci-lint-1.45.2-linux-arm64.deb -5157a58c8f9ab85c33af2e46f0d7c57a3b1e8953b81d61130e292e09f545cfab golangci-lint-1.45.2-linux-mips64le.tar.gz -518cd027644129fbf8ec4f02bd6f9ad7278aae826f92b63c80d4d0819ddde49a golangci-lint-1.45.2-linux-armv6.rpm -595ad6c6dade4c064351bc309f411703e457f8ffbb7a1806b3d8ee713333427f golangci-lint-1.45.2-linux-amd64.tar.gz -6994d6c80f0730751090986184a3481b4be2e6b6e84416238a2b857910045a4f golangci-lint-1.45.2-windows-arm64.zip -6c81652fc340118811b487f713c441fc6f527800bf5fd11b8929d08124efa015 golangci-lint-1.45.2-linux-armv7.tar.gz -726cb045559b7518bafdd3459de70a0647c087eb1b4634627a4b2e95b1258580 golangci-lint-1.45.2-freebsd-amd64.tar.gz -77df3774cdfda49b956d4a0e676da9a9b883f496ee37293c530770fef6b1d24e golangci-lint-1.45.2-linux-mips64.deb -7a9840f279a7d5d405bb434e101c2290964b3729630ac2add29280b962b7b9a5 golangci-lint-1.45.2-windows-armv6.zip -7d4bf9a5d80ec467aaaf66e78dbdcab567bbc6ba8151334c714eee58766aae32 golangci-lint-1.45.2-windows-armv7.zip -7e5f8821d39bb11d273b0841b34355f56bd5a45a2d5179f0d09e614e0efc0482 golangci-lint-1.45.2-linux-s390x.rpm -828de1bde796b23d8656b17a8885fbd879ef612795d62d1e4618126b419728b5 golangci-lint-1.45.2-linux-mips64.rpm -879a52107a797678a03c175cc7cf441411a14a01f66dc87f70bdfa304a4129a6 golangci-lint-1.45.2-windows-386.zip -87b6c7e3a3769f7d9abeb3bb82119b3c91e3c975300f6834fdeef8b2e37c98ff golangci-lint-1.45.2-linux-amd64.rpm -8b605c6d686c8af53ecc4ef39544541eeb1644d34cc10f9ffc5087808210c4ff golangci-lint-1.45.2-linux-s390x.deb -9427dbf51d0ac6f73a0f992838bd40c817470cc5bf6c8e2e2bea6fac46d7af6e golangci-lint-1.45.2-linux-ppc64le.tar.gz -995e509e895ca6a64ffc7395ac884d5961bdec98423cb896b17f345a9b4a19cf golangci-lint-1.45.2-darwin-amd64.tar.gz -a3f36278f2ea5516341e9071a2df6e65df272be80230b5406a12b72c6d425bee golangci-lint-1.45.2-linux-armv7.rpm -a5e12c50c23e87ac1deffc872f92ae85427b1198604969399805ae47cfe43f08 golangci-lint-1.45.2-linux-riscv64.tar.gz -aa8fa1be0729dbc2fbc4e01e82027097613eee74bd686ebef20f860b01fff8b3 golangci-lint-1.45.2-freebsd-armv6.tar.gz -c2b9669decc1b638cf2ee9060571af4e255f6dfcbb225c293e3a7ee4bb2c7217 golangci-lint-1.45.2-darwin-arm64.tar.gz -dfa8bdaf0387aec1cd5c1aa8857f67b2bbdfc2e42efce540c8fb9bbe3e8af302 golangci-lint-1.45.2-linux-armv6.tar.gz -eb8b8539dd017eee5c131ea9b875893ab2cebeeca41e8c6624907fb02224d643 golangci-lint-1.45.2-linux-386.rpm -ed6c7e17a857f30d715c5302fa250d95936936b277024bffea201187a257d7a7 golangci-lint-1.45.2-linux-armv6.deb -ef4d0154ace4001f01b288baeb118176242efb4fd163e178763e3213b77ef30b golangci-lint-1.45.2-linux-mips64le.deb -ef7002a2229f5ff5ba201a715fcf877664ea88decbe58e69d163293913024955 golangci-lint-1.45.2-linux-s390x.tar.gz -f13ecbd09228632e6bbe91a8324bd675c406eed22eb6d2c1e8192eed9ec4f914 golangci-lint-1.45.2-linux-386.tar.gz -f4cd9cfb09252f51699407277512263cae8409b665dd764f55a34738d0e89edc golangci-lint-1.45.2-linux-riscv64.rpm -fb1945dc59d37c9d14bf0a4aea11ea8651fa0e1d582ea80c4c44d0a536c08893 golangci-lint-1.45.2-linux-mips64.tar.gz -fe542c22738010f453c735a3c410decfd3784d1bd394b395c298ee298fc4c606 golangci-lint-1.45.2-linux-mips64le.rpm +658078aaaf7608693f37c4cf1380b2af418ab8b2d23fdb33e7e2d4339328590e golangci-lint-1.46.2-darwin-amd64.tar.gz +81f9b4afd62ec5e612ef8bc3b1d612a88b56ff289874831845cdad394427385f golangci-lint-1.46.2-darwin-arm64.tar.gz +943486e703e62ec55ecd90caeb22bcd39f8cc3962a93eec18c06b7bae12cb46f golangci-lint-1.46.2-freebsd-386.tar.gz +a75dd9ba7e08e8315c411697171db5375c0f6a1ece9e6fbeb9e9a4386822e17d golangci-lint-1.46.2-freebsd-amd64.tar.gz +83eedca1af72e8be055a1235177eb1b33524fbf08bec5730df2e6c3efade2b23 golangci-lint-1.46.2-freebsd-armv6.tar.gz +513d276c490de6f82baa01f9346d8d78b385f2ae97608f42f05d1f0f1314cd54 golangci-lint-1.46.2-freebsd-armv7.tar.gz +461a60016d516c69d406dc3e2d4957b722dbe684b7085dfac4802d0f84409e27 golangci-lint-1.46.2-linux-386.tar.gz +242cd4f2d6ac0556e315192e8555784d13da5d1874e51304711570769c4f2b9b golangci-lint-1.46.2-linux-amd64.tar.gz +ff5448ada2b3982581984d64b0dec614dba0a3ea4cab2d6a343c77927fc89f7e golangci-lint-1.46.2-linux-arm64.tar.gz +177f5210ef04aee282bfbc6ec519d36af5fb7d2b2c8d3f4ea5e59fdba71b0a27 golangci-lint-1.46.2-linux-armv6.tar.gz +10dd512a36ee978a1009edbca3ba3af410f0fda8df4d85f0e4793a24213870cc golangci-lint-1.46.2-linux-armv7.tar.gz +67779fa517c688c9db1090c3c456117d95c6b92979c623fe8cce8fb84251f21e golangci-lint-1.46.2-linux-mips64.tar.gz +c085f0f57bdccbb2c902a41b72ce210a3dfff16ca856789374745ab52004b6ee golangci-lint-1.46.2-linux-mips64le.tar.gz +abecef6421499248e58ed75d2938bc12b4b1f98b057f25060680b77bb51a881e golangci-lint-1.46.2-linux-ppc64le.tar.gz +134843a8f5c5c182c11979ea75f5866945d54757b2a04f3e5e04a0cf4fbf3a39 golangci-lint-1.46.2-linux-riscv64.tar.gz +9fe21a9476567aafe7a2e1a926b9641a39f920d4c0ea8eda9d968bc6136337f9 golangci-lint-1.46.2-linux-s390x.tar.gz +b48a421ec12a43f8fc8f977b9cf7d4a1ea1c4b97f803a238de7d3ce4ab23a84b golangci-lint-1.46.2-windows-386.zip +604acc1378a566abb0eac799362f3a37b7fcb5fa2268aeb2d5d954c829367301 golangci-lint-1.46.2-windows-amd64.zip +927def10db073da9687594072e6a3d9c891f67fa897105a2cfd715e018e7386c golangci-lint-1.46.2-windows-arm64.zip +729b76ed1d8b4e2612e38772b211503cb940e00a137bbaace1aa066f7c943737 golangci-lint-1.46.2-windows-armv6.zip +ea27c86d91e0b245ecbcfbf6cdb4ac0522d4bc6dca56bba02ea1bc77ad2917ac golangci-lint-1.46.2-windows-armv7.zip diff --git a/build/ci.go b/build/ci.go index 4129168be53a7..80b0400459397 100644 --- a/build/ci.go +++ b/build/ci.go @@ -132,12 +132,12 @@ var ( // Note: the following Ubuntu releases have been officially deprecated on Launchpad: // wily, yakkety, zesty, artful, cosmic, disco, eoan, groovy, hirsuite debDistroGoBoots = map[string]string{ - "trusty": "golang-1.11", // EOL: 04/2024 - "xenial": "golang-go", // EOL: 04/2026 - "bionic": "golang-go", // EOL: 04/2028 - "focal": "golang-go", // EOL: 04/2030 - "impish": "golang-go", // EOL: 07/2022 - "jammy": "golang-go", // EOL: 04/2032 + "trusty": "golang-1.11", // EOL: 04/2024 + "xenial": "golang-go", // EOL: 04/2026 + "bionic": "golang-go", // EOL: 04/2028 + "focal": "golang-go", // EOL: 04/2030 + "impish": "golang-go", // EOL: 07/2022 + "jammy": "golang-go", // EOL: 04/2032 //"kinetic": "golang-go", // EOL: 07/2023 } @@ -149,7 +149,7 @@ var ( // This is the version of go that will be downloaded by // // go run ci.go install -dlgo - dlgoVersion = "1.18.1" + dlgoVersion = "1.18.5" ) var GOBIN, _ = filepath.Abs(filepath.Join("build", "bin")) @@ -200,9 +200,10 @@ func main() { func doInstall(cmdline []string) { var ( - dlgo = flag.Bool("dlgo", false, "Download Go and build with it") - arch = flag.String("arch", "", "Architecture to cross build for") - cc = flag.String("cc", "", "C compiler to cross build with") + dlgo = flag.Bool("dlgo", false, "Download Go and build with it") + arch = flag.String("arch", "", "Architecture to cross build for") + cc = flag.String("cc", "", "C compiler to cross build with") + staticlink = flag.Bool("static", false, "Create statically-linked executable") ) flag.CommandLine.Parse(cmdline) @@ -213,9 +214,12 @@ func doInstall(cmdline []string) { tc.Root = build.DownloadGo(csdb, dlgoVersion) } + // Disable CLI markdown doc generation in release builds. + buildTags := []string{"urfave_cli_no_docs"} + // Configure the build. env := build.Env() - gobuild := tc.Go("build", buildFlags(env)...) + gobuild := tc.Go("build", buildFlags(env, *staticlink, buildTags)...) // arm64 CI builders are memory-constrained and can't handle concurrent builds, // better disable it. This check isn't the best, it should probably @@ -248,7 +252,7 @@ func doInstall(cmdline []string) { } // buildFlags returns the go tool flags for building. -func buildFlags(env build.Environment) (flags []string) { +func buildFlags(env build.Environment, staticLinking bool, buildTags []string) (flags []string) { var ld []string if env.Commit != "" { ld = append(ld, "-X", "main.gitCommit="+env.Commit) @@ -259,14 +263,24 @@ func buildFlags(env build.Environment) (flags []string) { if runtime.GOOS == "darwin" { ld = append(ld, "-s") } - // Enforce the stacksize to 8M, which is the case on most platforms apart from - // alpine Linux. if runtime.GOOS == "linux" { - ld = append(ld, "-extldflags", "-Wl,-z,stack-size=0x800000") + // Enforce the stacksize to 8M, which is the case on most platforms apart from + // alpine Linux. + extld := []string{"-Wl,-z,stack-size=0x800000"} + if staticLinking { + extld = append(extld, "-static") + // Under static linking, use of certain glibc features must be + // disabled to avoid shared library dependencies. + buildTags = append(buildTags, "osusergo", "netgo") + } + ld = append(ld, "-extldflags", "'"+strings.Join(extld, " ")+"'") } if len(ld) > 0 { flags = append(flags, "-ldflags", strings.Join(ld, " ")) } + if len(buildTags) > 0 { + flags = append(flags, "-tags", strings.Join(buildTags, ",")) + } return flags } @@ -333,7 +347,7 @@ func doLint(cmdline []string) { // downloadLinter downloads and unpacks golangci-lint. func downloadLinter(cachedir string) string { - const version = "1.45.2" + const version = "1.46.2" csdb := build.MustLoadChecksums("build/checksums.txt") arch := runtime.GOARCH @@ -594,7 +608,7 @@ func doDocker(cmdline []string) { } if mismatch { // Build numbers mismatching, retry in a short time to - // avoid concurrent failes in both publisher images. If + // avoid concurrent fails in both publisher images. If // however the retry failed too, it means the concurrent // builder is still crunching, let that do the publish. if i == 0 { diff --git a/build/deb/ethereum/completions/bash/geth b/build/deb/ethereum/completions/bash/geth new file mode 100755 index 0000000000000..a78952793efb1 --- /dev/null +++ b/build/deb/ethereum/completions/bash/geth @@ -0,0 +1,16 @@ +_geth_bash_autocomplete() { + if [[ "${COMP_WORDS[0]}" != "source" ]]; then + local cur opts base + COMPREPLY=() + cur="${COMP_WORDS[COMP_CWORD]}" + if [[ "$cur" == "-"* ]]; then + opts=$( ${COMP_WORDS[@]:0:$COMP_CWORD} ${cur} --generate-bash-completion ) + else + opts=$( ${COMP_WORDS[@]:0:$COMP_CWORD} --generate-bash-completion ) + fi + COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) + return 0 + fi +} + +complete -o bashdefault -o default -o nospace -F _geth_bash_autocomplete geth diff --git a/build/deb/ethereum/completions/zsh/_geth b/build/deb/ethereum/completions/zsh/_geth new file mode 100644 index 0000000000000..119794c532bda --- /dev/null +++ b/build/deb/ethereum/completions/zsh/_geth @@ -0,0 +1,18 @@ +_geth_zsh_autocomplete() { + local -a opts + local cur + cur=${words[-1]} + if [[ "$cur" == "-"* ]]; then + opts=("${(@f)$(${words[@]:0:#words[@]-1} ${cur} --generate-bash-completion)}") + else + opts=("${(@f)$(${words[@]:0:#words[@]-1} --generate-bash-completion)}") + fi + + if [[ "${opts[1]}" != "" ]]; then + _describe 'values' opts + else + _files + fi +} + +compdef _geth_zsh_autocomplete geth diff --git a/build/deb/ethereum/deb.install b/build/deb/ethereum/deb.install index e7666ce5fb6b4..5a624594b06ca 100644 --- a/build/deb/ethereum/deb.install +++ b/build/deb/ethereum/deb.install @@ -1 +1,5 @@ build/bin/{{.BinaryName}} usr/bin +{{- if eq .BinaryName "geth" }} +build/deb/ethereum/completions/bash/geth etc/bash_completion.d +build/deb/ethereum/completions/zsh/_geth usr/share/zsh/vendor-completions +{{end -}} diff --git a/cmd/abigen/main.go b/cmd/abigen/main.go index 8f255143c52fa..56ebfa9e9bb95 100644 --- a/cmd/abigen/main.go +++ b/cmd/abigen/main.go @@ -30,7 +30,7 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/internal/flags" "github.com/ethereum/go-ethereum/log" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) var ( @@ -39,42 +39,44 @@ var ( gitDate = "" app *cli.App +) +var ( // Flags needed by abigen - abiFlag = cli.StringFlag{ + abiFlag = &cli.StringFlag{ Name: "abi", Usage: "Path to the Ethereum contract ABI json to bind, - for STDIN", } - binFlag = cli.StringFlag{ + binFlag = &cli.StringFlag{ Name: "bin", Usage: "Path to the Ethereum contract bytecode (generate deploy method)", } - typeFlag = cli.StringFlag{ + typeFlag = &cli.StringFlag{ Name: "type", Usage: "Struct name for the binding (default = package name)", } - jsonFlag = cli.StringFlag{ + jsonFlag = &cli.StringFlag{ Name: "combined-json", - Usage: "Path to the combined-json file generated by compiler", + Usage: "Path to the combined-json file generated by compiler, - for STDIN", } - excFlag = cli.StringFlag{ + excFlag = &cli.StringFlag{ Name: "exc", Usage: "Comma separated types to exclude from binding", } - pkgFlag = cli.StringFlag{ + pkgFlag = &cli.StringFlag{ Name: "pkg", Usage: "Package name to generate the binding into", } - outFlag = cli.StringFlag{ + outFlag = &cli.StringFlag{ Name: "out", Usage: "Output file for the generated binding (default = stdout)", } - langFlag = cli.StringFlag{ + langFlag = &cli.StringFlag{ Name: "lang", Usage: "Destination language for the bindings (go, java, objc)", Value: "go", } - aliasFlag = cli.StringFlag{ + aliasFlag = &cli.StringFlag{ Name: "alias", Usage: "Comma separated aliases for function and event renaming, e.g. original1=alias1, original2=alias2", } @@ -82,6 +84,7 @@ var ( func init() { app = flags.NewApp(gitCommit, gitDate, "ethereum checkpoint helper tool") + app.Name = "abigen" app.Flags = []cli.Flag{ abiFlag, binFlag, @@ -93,17 +96,17 @@ func init() { langFlag, aliasFlag, } - app.Action = utils.MigrateFlags(abigen) - cli.CommandHelpTemplate = flags.OriginCommandHelpTemplate + app.Action = abigen } func abigen(c *cli.Context) error { utils.CheckExclusive(c, abiFlag, jsonFlag) // Only one source can be selected. - if c.GlobalString(pkgFlag.Name) == "" { + + if c.String(pkgFlag.Name) == "" { utils.Fatalf("No destination package specified (--pkg)") } var lang bind.Lang - switch c.GlobalString(langFlag.Name) { + switch c.String(langFlag.Name) { case "go": lang = bind.LangGo case "java": @@ -112,7 +115,7 @@ func abigen(c *cli.Context) error { lang = bind.LangObjC utils.Fatalf("Objc binding generation is uncompleted") default: - utils.Fatalf("Unsupported destination language \"%s\" (--lang)", c.GlobalString(langFlag.Name)) + utils.Fatalf("Unsupported destination language \"%s\" (--lang)", c.String(langFlag.Name)) } // If the entire solidity code was specified, build and bind based on that var ( @@ -123,13 +126,13 @@ func abigen(c *cli.Context) error { libs = make(map[string]string) aliases = make(map[string]string) ) - if c.GlobalString(abiFlag.Name) != "" { + if c.String(abiFlag.Name) != "" { // Load up the ABI, optional bytecode and type name from the parameters var ( abi []byte err error ) - input := c.GlobalString(abiFlag.Name) + input := c.String(abiFlag.Name) if input == "-" { abi, err = io.ReadAll(os.Stdin) } else { @@ -141,7 +144,7 @@ func abigen(c *cli.Context) error { abis = append(abis, string(abi)) var bin []byte - if binFile := c.GlobalString(binFlag.Name); binFile != "" { + if binFile := c.String(binFlag.Name); binFile != "" { if bin, err = os.ReadFile(binFile); err != nil { utils.Fatalf("Failed to read input bytecode: %v", err) } @@ -151,23 +154,32 @@ func abigen(c *cli.Context) error { } bins = append(bins, string(bin)) - kind := c.GlobalString(typeFlag.Name) + kind := c.String(typeFlag.Name) if kind == "" { - kind = c.GlobalString(pkgFlag.Name) + kind = c.String(pkgFlag.Name) } types = append(types, kind) } else { // Generate the list of types to exclude from binding exclude := make(map[string]bool) - for _, kind := range strings.Split(c.GlobalString(excFlag.Name), ",") { + for _, kind := range strings.Split(c.String(excFlag.Name), ",") { exclude[strings.ToLower(kind)] = true } var contracts map[string]*compiler.Contract - if c.GlobalIsSet(jsonFlag.Name) { - jsonOutput, err := os.ReadFile(c.GlobalString(jsonFlag.Name)) + if c.IsSet(jsonFlag.Name) { + var ( + input = c.String(jsonFlag.Name) + jsonOutput []byte + err error + ) + if input == "-" { + jsonOutput, err = io.ReadAll(os.Stdin) + } else { + jsonOutput, err = os.ReadFile(input) + } if err != nil { - utils.Fatalf("Failed to read combined-json from compiler: %v", err) + utils.Fatalf("Failed to read combined-json: %v", err) } contracts, err = compiler.ParseCombinedJSON(jsonOutput, "", "", "", "") if err != nil { @@ -189,33 +201,37 @@ func abigen(c *cli.Context) error { nameParts := strings.Split(name, ":") types = append(types, nameParts[len(nameParts)-1]) - libPattern := crypto.Keccak256Hash([]byte(name)).String()[2:36] + // Derive the library placeholder which is a 34 character prefix of the + // hex encoding of the keccak256 hash of the fully qualified library name. + // Note that the fully qualified library name is the path of its source + // file and the library name separated by ":". + libPattern := crypto.Keccak256Hash([]byte(name)).String()[2:36] // the first 2 chars are 0x libs[libPattern] = nameParts[len(nameParts)-1] } } // Extract all aliases from the flags - if c.GlobalIsSet(aliasFlag.Name) { + if c.IsSet(aliasFlag.Name) { // We support multi-versions for aliasing // e.g. // foo=bar,foo2=bar2 // foo:bar,foo2:bar2 re := regexp.MustCompile(`(?:(\w+)[:=](\w+))`) - submatches := re.FindAllStringSubmatch(c.GlobalString(aliasFlag.Name), -1) + submatches := re.FindAllStringSubmatch(c.String(aliasFlag.Name), -1) for _, match := range submatches { aliases[match[1]] = match[2] } } // Generate the contract binding - code, err := bind.Bind(types, abis, bins, sigs, c.GlobalString(pkgFlag.Name), lang, libs, aliases) + code, err := bind.Bind(types, abis, bins, sigs, c.String(pkgFlag.Name), lang, libs, aliases) if err != nil { utils.Fatalf("Failed to generate ABI binding: %v", err) } // Either flush it out to a file or display on the standard output - if !c.GlobalIsSet(outFlag.Name) { + if !c.IsSet(outFlag.Name) { fmt.Printf("%s\n", code) return nil } - if err := os.WriteFile(c.GlobalString(outFlag.Name), []byte(code), 0600); err != nil { + if err := os.WriteFile(c.String(outFlag.Name), []byte(code), 0600); err != nil { utils.Fatalf("Failed to write ABI binding: %v", err) } return nil diff --git a/cmd/checkpoint-admin/common.go b/cmd/checkpoint-admin/common.go index 05a45dfbf9970..f86ac24f06c19 100644 --- a/cmd/checkpoint-admin/common.go +++ b/cmd/checkpoint-admin/common.go @@ -28,12 +28,12 @@ import ( "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) // newClient creates a client with specified remote URL. func newClient(ctx *cli.Context) *ethclient.Client { - client, err := ethclient.Dial(ctx.GlobalString(nodeURLFlag.Name)) + client, err := ethclient.Dial(ctx.String(nodeURLFlag.Name)) if err != nil { utils.Fatalf("Failed to connect to Ethereum node: %v", err) } @@ -64,9 +64,9 @@ func getContractAddr(client *rpc.Client) common.Address { func getCheckpoint(ctx *cli.Context, client *rpc.Client) *params.TrustedCheckpoint { var checkpoint *params.TrustedCheckpoint - if ctx.GlobalIsSet(indexFlag.Name) { + if ctx.IsSet(indexFlag.Name) { var result [3]string - index := uint64(ctx.GlobalInt64(indexFlag.Name)) + index := uint64(ctx.Int64(indexFlag.Name)) if err := client.Call(&result, "les_getCheckpoint", index); err != nil { utils.Fatalf("Failed to get local checkpoint %v, please ensure the les API is exposed", err) } diff --git a/cmd/checkpoint-admin/exec.go b/cmd/checkpoint-admin/exec.go index 352a96d9e6f00..cb67d0306d43a 100644 --- a/cmd/checkpoint-admin/exec.go +++ b/cmd/checkpoint-admin/exec.go @@ -36,10 +36,10 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) -var commandDeploy = cli.Command{ +var commandDeploy = &cli.Command{ Name: "deploy", Usage: "Deploy a new checkpoint oracle contract", Flags: []cli.Flag{ @@ -49,10 +49,10 @@ var commandDeploy = cli.Command{ signersFlag, thresholdFlag, }, - Action: utils.MigrateFlags(deploy), + Action: deploy, } -var commandSign = cli.Command{ +var commandSign = &cli.Command{ Name: "sign", Usage: "Sign the checkpoint with the specified key", Flags: []cli.Flag{ @@ -63,10 +63,10 @@ var commandSign = cli.Command{ hashFlag, oracleFlag, }, - Action: utils.MigrateFlags(sign), + Action: sign, } -var commandPublish = cli.Command{ +var commandPublish = &cli.Command{ Name: "publish", Usage: "Publish a checkpoint into the oracle", Flags: []cli.Flag{ @@ -76,7 +76,7 @@ var commandPublish = cli.Command{ indexFlag, signaturesFlag, }, - Action: utils.MigrateFlags(publish), + Action: publish, } // deploy deploys the checkpoint registrar contract. @@ -132,7 +132,7 @@ func sign(ctx *cli.Context) error { node *rpc.Client oracle *checkpointoracle.CheckpointOracle ) - if !ctx.GlobalIsSet(nodeURLFlag.Name) { + if !ctx.IsSet(nodeURLFlag.Name) { // Offline mode signing offline = true if !ctx.IsSet(hashFlag.Name) { @@ -151,7 +151,7 @@ func sign(ctx *cli.Context) error { address = common.HexToAddress(ctx.String(oracleFlag.Name)) } else { // Interactive mode signing, retrieve the data from the remote node - node = newRPCClient(ctx.GlobalString(nodeURLFlag.Name)) + node = newRPCClient(ctx.String(nodeURLFlag.Name)) checkpoint := getCheckpoint(ctx, node) chash, cindex, address = checkpoint.Hash(), checkpoint.SectionIndex, getContractAddr(node) @@ -265,7 +265,7 @@ func publish(ctx *cli.Context) error { } // Retrieve the checkpoint we want to sign to sort the signatures var ( - client = newRPCClient(ctx.GlobalString(nodeURLFlag.Name)) + client = newRPCClient(ctx.String(nodeURLFlag.Name)) addr, oracle = newContract(client) checkpoint = getCheckpoint(ctx, client) sighash = sighash(checkpoint.SectionIndex, addr, checkpoint.Hash()) diff --git a/cmd/checkpoint-admin/main.go b/cmd/checkpoint-admin/main.go index 0fb5532147787..0604ccaad622f 100644 --- a/cmd/checkpoint-admin/main.go +++ b/cmd/checkpoint-admin/main.go @@ -25,20 +25,20 @@ import ( "github.com/ethereum/go-ethereum/common/fdlimit" "github.com/ethereum/go-ethereum/internal/flags" "github.com/ethereum/go-ethereum/log" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) var ( // Git SHA1 commit hash of the release (set via linker flags) gitCommit = "" gitDate = "" -) -var app *cli.App + app *cli.App +) func init() { app = flags.NewApp(gitCommit, gitDate, "ethereum checkpoint helper tool") - app.Commands = []cli.Command{ + app.Commands = []*cli.Command{ commandStatus, commandDeploy, commandSign, @@ -48,46 +48,45 @@ func init() { oracleFlag, nodeURLFlag, } - cli.CommandHelpTemplate = flags.OriginCommandHelpTemplate } // Commonly used command line flags. var ( - indexFlag = cli.Int64Flag{ + indexFlag = &cli.Int64Flag{ Name: "index", Usage: "Checkpoint index (query latest from remote node if not specified)", } - hashFlag = cli.StringFlag{ + hashFlag = &cli.StringFlag{ Name: "hash", Usage: "Checkpoint hash (query latest from remote node if not specified)", } - oracleFlag = cli.StringFlag{ + oracleFlag = &cli.StringFlag{ Name: "oracle", Usage: "Checkpoint oracle address (query from remote node if not specified)", } - thresholdFlag = cli.Int64Flag{ + thresholdFlag = &cli.Int64Flag{ Name: "threshold", Usage: "Minimal number of signatures required to approve a checkpoint", } - nodeURLFlag = cli.StringFlag{ + nodeURLFlag = &cli.StringFlag{ Name: "rpc", Value: "http://localhost:8545", Usage: "The rpc endpoint of a local or remote geth node", } - clefURLFlag = cli.StringFlag{ + clefURLFlag = &cli.StringFlag{ Name: "clef", Value: "http://localhost:8550", Usage: "The rpc endpoint of clef", } - signerFlag = cli.StringFlag{ + signerFlag = &cli.StringFlag{ Name: "signer", Usage: "Signer address for clef signing", } - signersFlag = cli.StringFlag{ + signersFlag = &cli.StringFlag{ Name: "signers", Usage: "Comma separated accounts of trusted checkpoint signers", } - signaturesFlag = cli.StringFlag{ + signaturesFlag = &cli.StringFlag{ Name: "signatures", Usage: "Comma separated checkpoint signatures to submit", } diff --git a/cmd/checkpoint-admin/status.go b/cmd/checkpoint-admin/status.go index f613501eb35d6..bec97aed12bd5 100644 --- a/cmd/checkpoint-admin/status.go +++ b/cmd/checkpoint-admin/status.go @@ -19,24 +19,23 @@ package main import ( "fmt" - "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/common" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) -var commandStatus = cli.Command{ +var commandStatus = &cli.Command{ Name: "status", Usage: "Fetches the signers and checkpoint status of the oracle contract", Flags: []cli.Flag{ nodeURLFlag, }, - Action: utils.MigrateFlags(status), + Action: status, } // status fetches the admin list of specified registrar contract. func status(ctx *cli.Context) error { // Create a wrapper around the checkpoint oracle contract - addr, oracle := newContract(newRPCClient(ctx.GlobalString(nodeURLFlag.Name))) + addr, oracle := newContract(newRPCClient(ctx.String(nodeURLFlag.Name))) fmt.Printf("Oracle => %s\n", addr.Hex()) fmt.Println() diff --git a/cmd/clef/main.go b/cmd/clef/main.go index b1ffa38ffefaa..05290f52feb89 100644 --- a/cmd/clef/main.go +++ b/cmd/clef/main.go @@ -30,7 +30,6 @@ import ( "os/signal" "path/filepath" "runtime" - "sort" "strings" "time" @@ -55,7 +54,7 @@ import ( "github.com/ethereum/go-ethereum/signer/storage" "github.com/mattn/go-colorable" "github.com/mattn/go-isatty" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) const legalWarning = ` @@ -73,70 +72,70 @@ PURPOSE. See the GNU General Public License for more details. ` var ( - logLevelFlag = cli.IntFlag{ + logLevelFlag = &cli.IntFlag{ Name: "loglevel", Value: 4, Usage: "log level to emit to the screen", } - advancedMode = cli.BoolFlag{ + advancedMode = &cli.BoolFlag{ Name: "advanced", Usage: "If enabled, issues warnings instead of rejections for suspicious requests. Default off", } - acceptFlag = cli.BoolFlag{ + acceptFlag = &cli.BoolFlag{ Name: "suppress-bootwarn", Usage: "If set, does not show the warning during boot", } - keystoreFlag = cli.StringFlag{ + keystoreFlag = &cli.StringFlag{ Name: "keystore", Value: filepath.Join(node.DefaultDataDir(), "keystore"), Usage: "Directory for the keystore", } - configdirFlag = cli.StringFlag{ + configdirFlag = &cli.StringFlag{ Name: "configdir", Value: DefaultConfigDir(), Usage: "Directory for Clef configuration", } - chainIdFlag = cli.Int64Flag{ + chainIdFlag = &cli.Int64Flag{ Name: "chainid", Value: params.MainnetChainConfig.ChainID.Int64(), Usage: "Chain id to use for signing (1=mainnet, 3=Ropsten, 4=Rinkeby, 5=Goerli)", } - rpcPortFlag = cli.IntFlag{ - Name: "http.port", - Usage: "HTTP-RPC server listening port", - Value: node.DefaultHTTPPort + 5, + rpcPortFlag = &cli.IntFlag{ + Name: "http.port", + Usage: "HTTP-RPC server listening port", + Value: node.DefaultHTTPPort + 5, + Category: flags.APICategory, } - signerSecretFlag = cli.StringFlag{ + signerSecretFlag = &cli.StringFlag{ Name: "signersecret", Usage: "A file containing the (encrypted) master seed to encrypt Clef data, e.g. keystore credentials and ruleset hash", } - customDBFlag = cli.StringFlag{ + customDBFlag = &cli.StringFlag{ Name: "4bytedb-custom", Usage: "File used for writing new 4byte-identifiers submitted via API", Value: "./4byte-custom.json", } - auditLogFlag = cli.StringFlag{ + auditLogFlag = &cli.StringFlag{ Name: "auditlog", Usage: "File used to emit audit logs. Set to \"\" to disable", Value: "audit.log", } - ruleFlag = cli.StringFlag{ + ruleFlag = &cli.StringFlag{ Name: "rules", Usage: "Path to the rule file to auto-authorize requests with", } - stdiouiFlag = cli.BoolFlag{ + stdiouiFlag = &cli.BoolFlag{ Name: "stdio-ui", Usage: "Use STDIN/STDOUT as a channel for an external UI. " + "This means that an STDIN/STDOUT is used for RPC-communication with a e.g. a graphical user " + "interface, and can be used when Clef is started by an external process.", } - testFlag = cli.BoolFlag{ + testFlag = &cli.BoolFlag{ Name: "stdio-ui-test", Usage: "Mechanism to test interface between Clef and UI. Requires 'stdio-ui'.", } - app = cli.NewApp() - initCommand = cli.Command{ - Action: utils.MigrateFlags(initializeSecrets), + initCommand = &cli.Command{ + Action: initializeSecrets, Name: "init", Usage: "Initialize the signer, generate secret storage", ArgsUsage: "", @@ -148,8 +147,8 @@ var ( The init command generates a master seed which Clef can use to store credentials and data needed for the rule-engine to work.`, } - attestCommand = cli.Command{ - Action: utils.MigrateFlags(attestFile), + attestCommand = &cli.Command{ + Action: attestFile, Name: "attest", Usage: "Attest that a js-file is to be used", ArgsUsage: "", @@ -165,8 +164,8 @@ incoming requests. Whenever you make an edit to the rule file, you need to use attestation to tell Clef that the file is 'safe' to execute.`, } - setCredentialCommand = cli.Command{ - Action: utils.MigrateFlags(setCredential), + setCredentialCommand = &cli.Command{ + Action: setCredential, Name: "setpw", Usage: "Store a credential for a keystore file", ArgsUsage: "
", @@ -178,8 +177,8 @@ Clef that the file is 'safe' to execute.`, Description: ` The setpw command stores a password for a given address (keyfile). `} - delCredentialCommand = cli.Command{ - Action: utils.MigrateFlags(removeCredential), + delCredentialCommand = &cli.Command{ + Action: removeCredential, Name: "delpw", Usage: "Remove a credential for a keystore file", ArgsUsage: "
", @@ -191,8 +190,8 @@ The setpw command stores a password for a given address (keyfile). Description: ` The delpw command removes a password for a given address (keyfile). `} - newAccountCommand = cli.Command{ - Action: utils.MigrateFlags(newAccount), + newAccountCommand = &cli.Command{ + Action: newAccount, Name: "newaccount", Usage: "Create a new account", ArgsUsage: "", @@ -207,7 +206,7 @@ The newaccount command creates a new keystore-backed account. It is a convenienc which can be used in lieu of an external UI.`, } - gendocCommand = cli.Command{ + gendocCommand = &cli.Command{ Action: GenDoc, Name: "gendoc", Usage: "Generate documentation about json-rpc format", @@ -216,39 +215,16 @@ The gendoc generates example structures of the json-rpc communication types. `} ) -// AppHelpFlagGroups is the application flags, grouped by functionality. -var AppHelpFlagGroups = []flags.FlagGroup{ - { - Name: "FLAGS", - Flags: []cli.Flag{ - logLevelFlag, - keystoreFlag, - configdirFlag, - chainIdFlag, - utils.LightKDFFlag, - utils.NoUSBFlag, - utils.SmartCardDaemonPathFlag, - utils.HTTPListenAddrFlag, - utils.HTTPVirtualHostsFlag, - utils.IPCDisabledFlag, - utils.IPCPathFlag, - utils.HTTPEnabledFlag, - rpcPortFlag, - signerSecretFlag, - customDBFlag, - auditLogFlag, - ruleFlag, - stdiouiFlag, - testFlag, - advancedMode, - acceptFlag, - }, - }, -} +var ( + // Git SHA1 commit hash of the release (set via linker flags) + gitCommit = "" + gitDate = "" + + app = flags.NewApp(gitCommit, gitDate, "Manage Ethereum account operations") +) func init() { app.Name = "Clef" - app.Usage = "Manage Ethereum account operations" app.Flags = []cli.Flag{ logLevelFlag, keystoreFlag, @@ -273,46 +249,12 @@ func init() { acceptFlag, } app.Action = signer - app.Commands = []cli.Command{initCommand, + app.Commands = []*cli.Command{initCommand, attestCommand, setCredentialCommand, delCredentialCommand, newAccountCommand, - gendocCommand} - cli.CommandHelpTemplate = flags.CommandHelpTemplate - // Override the default app help template - cli.AppHelpTemplate = flags.ClefAppHelpTemplate - - // Override the default app help printer, but only for the global app help - originalHelpPrinter := cli.HelpPrinter - cli.HelpPrinter = func(w io.Writer, tmpl string, data interface{}) { - if tmpl == flags.ClefAppHelpTemplate { - // Render out custom usage screen - originalHelpPrinter(w, tmpl, flags.HelpData{App: data, FlagGroups: AppHelpFlagGroups}) - } else if tmpl == flags.CommandHelpTemplate { - // Iterate over all command specific flags and categorize them - categorized := make(map[string][]cli.Flag) - for _, flag := range data.(cli.Command).Flags { - if _, ok := categorized[flag.String()]; !ok { - categorized[flags.FlagCategory(flag, AppHelpFlagGroups)] = append(categorized[flags.FlagCategory(flag, AppHelpFlagGroups)], flag) - } - } - - // sort to get a stable ordering - sorted := make([]flags.FlagGroup, 0, len(categorized)) - for cat, flgs := range categorized { - sorted = append(sorted, flags.FlagGroup{Name: cat, Flags: flgs}) - } - sort.Sort(flags.ByCategory(sorted)) - - // add sorted array to data and render with default printer - originalHelpPrinter(w, tmpl, map[string]interface{}{ - "cmd": data, - "categorizedFlags": sorted, - }) - } else { - originalHelpPrinter(w, tmpl, data) - } + gendocCommand, } } @@ -329,7 +271,7 @@ func initializeSecrets(c *cli.Context) error { return err } // Ensure the master key does not yet exist, we're not willing to overwrite - configDir := c.GlobalString(configdirFlag.Name) + configDir := c.String(configdirFlag.Name) if err := os.Mkdir(configDir, 0700); err != nil && !os.IsExist(err) { return err } @@ -347,7 +289,7 @@ func initializeSecrets(c *cli.Context) error { return fmt.Errorf("failed to read enough random") } n, p := keystore.StandardScryptN, keystore.StandardScryptP - if c.GlobalBool(utils.LightKDFFlag.Name) { + if c.Bool(utils.LightKDFFlag.Name) { n, p = keystore.LightScryptN, keystore.LightScryptP } text := "The master seed of clef will be locked with a password.\nPlease specify a password. Do not forget this password!" @@ -390,8 +332,9 @@ You should treat 'masterseed.json' with utmost secrecy and make a backup of it! `) return nil } + func attestFile(ctx *cli.Context) error { - if len(ctx.Args()) < 1 { + if ctx.NArg() < 1 { utils.Fatalf("This command requires an argument.") } if err := initialize(ctx); err != nil { @@ -402,7 +345,7 @@ func attestFile(ctx *cli.Context) error { if err != nil { utils.Fatalf(err.Error()) } - configDir := ctx.GlobalString(configdirFlag.Name) + configDir := ctx.String(configdirFlag.Name) vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), stretchedKey)[:10])) confKey := crypto.Keccak256([]byte("config"), stretchedKey) @@ -415,7 +358,7 @@ func attestFile(ctx *cli.Context) error { } func setCredential(ctx *cli.Context) error { - if len(ctx.Args()) < 1 { + if ctx.NArg() < 1 { utils.Fatalf("This command requires an address to be passed as an argument") } if err := initialize(ctx); err != nil { @@ -433,7 +376,7 @@ func setCredential(ctx *cli.Context) error { if err != nil { utils.Fatalf(err.Error()) } - configDir := ctx.GlobalString(configdirFlag.Name) + configDir := ctx.String(configdirFlag.Name) vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), stretchedKey)[:10])) pwkey := crypto.Keccak256([]byte("credentials"), stretchedKey) @@ -445,7 +388,7 @@ func setCredential(ctx *cli.Context) error { } func removeCredential(ctx *cli.Context) error { - if len(ctx.Args()) < 1 { + if ctx.NArg() < 1 { utils.Fatalf("This command requires an address to be passed as an argument") } if err := initialize(ctx); err != nil { @@ -461,7 +404,7 @@ func removeCredential(ctx *cli.Context) error { if err != nil { utils.Fatalf(err.Error()) } - configDir := ctx.GlobalString(configdirFlag.Name) + configDir := ctx.String(configdirFlag.Name) vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), stretchedKey)[:10])) pwkey := crypto.Keccak256([]byte("credentials"), stretchedKey) @@ -481,8 +424,8 @@ func newAccount(c *cli.Context) error { var ( ui = core.NewCommandlineUI() pwStorage storage.Storage = &storage.NoStorage{} - ksLoc = c.GlobalString(keystoreFlag.Name) - lightKdf = c.GlobalBool(utils.LightKDFFlag.Name) + ksLoc = c.String(keystoreFlag.Name) + lightKdf = c.Bool(utils.LightKDFFlag.Name) ) log.Info("Starting clef", "keystore", ksLoc, "light-kdf", lightKdf) am := core.StartClefAccountManager(ksLoc, true, lightKdf, "") @@ -500,13 +443,13 @@ func newAccount(c *cli.Context) error { func initialize(c *cli.Context) error { // Set up the logger to print everything logOutput := os.Stdout - if c.GlobalBool(stdiouiFlag.Name) { + if c.Bool(stdiouiFlag.Name) { logOutput = os.Stderr // If using the stdioui, we can't do the 'confirm'-flow - if !c.GlobalBool(acceptFlag.Name) { + if !c.Bool(acceptFlag.Name) { fmt.Fprint(logOutput, legalWarning) } - } else if !c.GlobalBool(acceptFlag.Name) { + } else if !c.Bool(acceptFlag.Name) { if !confirm(legalWarning) { return fmt.Errorf("aborted by user") } @@ -545,8 +488,8 @@ func ipcEndpoint(ipcPath, datadir string) string { func signer(c *cli.Context) error { // If we have some unrecognized command, bail out - if args := c.Args(); len(args) > 0 { - return fmt.Errorf("invalid command: %q", args[0]) + if c.NArg() > 0 { + return fmt.Errorf("invalid command: %q", c.Args().First()) } if err := initialize(c); err != nil { return err @@ -554,7 +497,7 @@ func signer(c *cli.Context) error { var ( ui core.UIClientAPI ) - if c.GlobalBool(stdiouiFlag.Name) { + if c.Bool(stdiouiFlag.Name) { log.Info("Using stdin/stdout as UI-channel") ui = core.NewStdIOUI() } else { @@ -562,7 +505,7 @@ func signer(c *cli.Context) error { ui = core.NewCommandlineUI() } // 4bytedb data - fourByteLocal := c.GlobalString(customDBFlag.Name) + fourByteLocal := c.String(customDBFlag.Name) db, err := fourbyte.NewWithFile(fourByteLocal) if err != nil { utils.Fatalf(err.Error()) @@ -574,7 +517,7 @@ func signer(c *cli.Context) error { api core.ExternalAPI pwStorage storage.Storage = &storage.NoStorage{} ) - configDir := c.GlobalString(configdirFlag.Name) + configDir := c.String(configdirFlag.Name) if stretchedKey, err := readMasterKey(c, ui); err != nil { log.Warn("Failed to open master, rules disabled", "err", err) } else { @@ -591,7 +534,7 @@ func signer(c *cli.Context) error { configStorage := storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "config.json"), confkey) // Do we have a rule-file? - if ruleFile := c.GlobalString(ruleFlag.Name); ruleFile != "" { + if ruleFile := c.String(ruleFlag.Name); ruleFile != "" { ruleJS, err := os.ReadFile(ruleFile) if err != nil { log.Warn("Could not load rules, disabling", "file", ruleFile, "err", err) @@ -615,12 +558,12 @@ func signer(c *cli.Context) error { } } var ( - chainId = c.GlobalInt64(chainIdFlag.Name) - ksLoc = c.GlobalString(keystoreFlag.Name) - lightKdf = c.GlobalBool(utils.LightKDFFlag.Name) - advanced = c.GlobalBool(advancedMode.Name) - nousb = c.GlobalBool(utils.NoUSBFlag.Name) - scpath = c.GlobalString(utils.SmartCardDaemonPathFlag.Name) + chainId = c.Int64(chainIdFlag.Name) + ksLoc = c.String(keystoreFlag.Name) + lightKdf = c.Bool(utils.LightKDFFlag.Name) + advanced = c.Bool(advancedMode.Name) + nousb = c.Bool(utils.NoUSBFlag.Name) + scpath = c.String(utils.SmartCardDaemonPathFlag.Name) ) log.Info("Starting signer", "chainid", chainId, "keystore", ksLoc, "light-kdf", lightKdf, "advanced", advanced) @@ -632,7 +575,7 @@ func signer(c *cli.Context) error { ui.RegisterUIServer(core.NewUIServerAPI(apiImpl)) api = apiImpl // Audit logging - if logfile := c.GlobalString(auditLogFlag.Name); logfile != "" { + if logfile := c.String(auditLogFlag.Name); logfile != "" { api, err = core.NewAuditLogger(logfile, api) if err != nil { utils.Fatalf(err.Error()) @@ -647,16 +590,15 @@ func signer(c *cli.Context) error { rpcAPI := []rpc.API{ { Namespace: "account", - Public: true, Service: api, - Version: "1.0"}, + }, } - if c.GlobalBool(utils.HTTPEnabledFlag.Name) { - vhosts := utils.SplitAndTrim(c.GlobalString(utils.HTTPVirtualHostsFlag.Name)) - cors := utils.SplitAndTrim(c.GlobalString(utils.HTTPCORSDomainFlag.Name)) + if c.Bool(utils.HTTPEnabledFlag.Name) { + vhosts := utils.SplitAndTrim(c.String(utils.HTTPVirtualHostsFlag.Name)) + cors := utils.SplitAndTrim(c.String(utils.HTTPCORSDomainFlag.Name)) srv := rpc.NewServer() - err := node.RegisterApis(rpcAPI, []string{"account"}, srv, false) + err := node.RegisterApis(rpcAPI, []string{"account"}, srv) if err != nil { utils.Fatalf("Could not register API: %w", err) } @@ -666,7 +608,7 @@ func signer(c *cli.Context) error { port := c.Int(rpcPortFlag.Name) // start http server - httpEndpoint := fmt.Sprintf("%s:%d", c.GlobalString(utils.HTTPListenAddrFlag.Name), port) + httpEndpoint := fmt.Sprintf("%s:%d", c.String(utils.HTTPListenAddrFlag.Name), port) httpServer, addr, err := node.StartHTTPEndpoint(httpEndpoint, rpc.DefaultHTTPTimeouts, handler) if err != nil { utils.Fatalf("Could not start RPC api: %v", err) @@ -680,8 +622,8 @@ func signer(c *cli.Context) error { log.Info("HTTP endpoint closed", "url", extapiURL) }() } - if !c.GlobalBool(utils.IPCDisabledFlag.Name) { - givenPath := c.GlobalString(utils.IPCPathFlag.Name) + if !c.Bool(utils.IPCDisabledFlag.Name) { + givenPath := c.String(utils.IPCPathFlag.Name) ipcapiURL = ipcEndpoint(filepath.Join(givenPath, "clef.ipc"), configDir) listener, _, err := rpc.StartIPCEndpoint(ipcapiURL, rpcAPI) if err != nil { @@ -694,7 +636,7 @@ func signer(c *cli.Context) error { }() } - if c.GlobalBool(testFlag.Name) { + if c.Bool(testFlag.Name) { log.Info("Performing UI test") go testExternalUI(apiImpl) } @@ -720,7 +662,7 @@ func signer(c *cli.Context) error { // persistence requirements. func DefaultConfigDir() string { // Try to place the data folder in the user's home dir - home := utils.HomeDir() + home := flags.HomeDir() if home != "" { if runtime.GOOS == "darwin" { return filepath.Join(home, "Library", "Signer") @@ -740,10 +682,10 @@ func DefaultConfigDir() string { func readMasterKey(ctx *cli.Context, ui core.UIClientAPI) ([]byte, error) { var ( file string - configDir = ctx.GlobalString(configdirFlag.Name) + configDir = ctx.String(configdirFlag.Name) ) - if ctx.GlobalIsSet(signerSecretFlag.Name) { - file = ctx.GlobalString(signerSecretFlag.Name) + if ctx.IsSet(signerSecretFlag.Name) { + file = ctx.String(signerSecretFlag.Name) } else { file = filepath.Join(configDir, "masterseed.json") } @@ -817,7 +759,6 @@ func confirm(text string) bool { } func testExternalUI(api *core.SignerAPI) { - ctx := context.WithValue(context.Background(), "remote", "clef binary") ctx = context.WithValue(ctx, "scheme", "in-proc") ctx = context.WithValue(ctx, "local", "main") @@ -917,7 +858,6 @@ func testExternalUI(api *core.SignerAPI) { expectDeny("signdata - text", err) } { // Sign transaction - api.UI.ShowInfo("Please reject next transaction") time.Sleep(delay) data := hexutil.Bytes([]byte{}) @@ -960,7 +900,6 @@ func testExternalUI(api *core.SignerAPI) { } result := fmt.Sprintf("Tests completed. %d errors:\n%s\n", len(errs), strings.Join(errs, "\n")) api.UI.ShowInfo(result) - } type encryptedSeedStorage struct { @@ -996,8 +935,7 @@ func decryptSeed(keyjson []byte, auth string) ([]byte, error) { } // GenDoc outputs examples of all structures used in json-rpc communication -func GenDoc(ctx *cli.Context) { - +func GenDoc(ctx *cli.Context) error { var ( a = common.HexToAddress("0xdeadbeef000000000000000000000000deadbeef") b = common.HexToAddress("0x1111111122222222222233333333334444444444") @@ -1107,7 +1045,6 @@ func GenDoc(ctx *cli.Context) { var tx types.Transaction tx.UnmarshalBinary(rlpdata) add("OnApproved - SignTransactionResult", desc, ðapi.SignTransactionResult{Raw: rlpdata, Tx: &tx}) - } { // User input add("UserInputRequest", "Sent when clef needs the user to provide data. If 'password' is true, the input field should be treated accordingly (echo-free)", @@ -1146,4 +1083,5 @@ These data types are defined in the channel between clef and the UI`) for _, elem := range output { fmt.Println(elem) } + return nil } diff --git a/cmd/devp2p/discv4cmd.go b/cmd/devp2p/discv4cmd.go index 3b6dc09a1cc82..9d35880b128b7 100644 --- a/cmd/devp2p/discv4cmd.go +++ b/cmd/devp2p/discv4cmd.go @@ -25,17 +25,18 @@ import ( "github.com/ethereum/go-ethereum/cmd/devp2p/internal/v4test" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/internal/flags" "github.com/ethereum/go-ethereum/p2p/discover" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/params" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) var ( - discv4Command = cli.Command{ + discv4Command = &cli.Command{ Name: "discv4", Usage: "Node Discovery v4 tools", - Subcommands: []cli.Command{ + Subcommands: []*cli.Command{ discv4PingCommand, discv4RequestRecordCommand, discv4ResolveCommand, @@ -44,39 +45,41 @@ var ( discv4TestCommand, }, } - discv4PingCommand = cli.Command{ + discv4PingCommand = &cli.Command{ Name: "ping", Usage: "Sends ping to a node", Action: discv4Ping, ArgsUsage: "", + Flags: v4NodeFlags, } - discv4RequestRecordCommand = cli.Command{ + discv4RequestRecordCommand = &cli.Command{ Name: "requestenr", Usage: "Requests a node record using EIP-868 enrRequest", Action: discv4RequestRecord, ArgsUsage: "", + Flags: v4NodeFlags, } - discv4ResolveCommand = cli.Command{ + discv4ResolveCommand = &cli.Command{ Name: "resolve", Usage: "Finds a node in the DHT", Action: discv4Resolve, ArgsUsage: "", - Flags: []cli.Flag{bootnodesFlag}, + Flags: v4NodeFlags, } - discv4ResolveJSONCommand = cli.Command{ + discv4ResolveJSONCommand = &cli.Command{ Name: "resolve-json", Usage: "Re-resolves nodes in a nodes.json file", Action: discv4ResolveJSON, - Flags: []cli.Flag{bootnodesFlag}, + Flags: v4NodeFlags, ArgsUsage: "", } - discv4CrawlCommand = cli.Command{ + discv4CrawlCommand = &cli.Command{ Name: "crawl", Usage: "Updates a nodes.json file with random nodes found in the DHT", Action: discv4Crawl, - Flags: []cli.Flag{bootnodesFlag, crawlTimeoutFlag}, + Flags: flags.Merge(v4NodeFlags, []cli.Flag{crawlTimeoutFlag}), } - discv4TestCommand = cli.Command{ + discv4TestCommand = &cli.Command{ Name: "test", Usage: "Runs tests against a node", Action: discv4Test, @@ -91,34 +94,41 @@ var ( ) var ( - bootnodesFlag = cli.StringFlag{ + bootnodesFlag = &cli.StringFlag{ Name: "bootnodes", Usage: "Comma separated nodes used for bootstrapping", } - nodekeyFlag = cli.StringFlag{ + nodekeyFlag = &cli.StringFlag{ Name: "nodekey", Usage: "Hex-encoded node key", } - nodedbFlag = cli.StringFlag{ + nodedbFlag = &cli.StringFlag{ Name: "nodedb", Usage: "Nodes database location", } - listenAddrFlag = cli.StringFlag{ + listenAddrFlag = &cli.StringFlag{ Name: "addr", Usage: "Listening address", } - crawlTimeoutFlag = cli.DurationFlag{ + crawlTimeoutFlag = &cli.DurationFlag{ Name: "timeout", Usage: "Time limit for the crawl.", Value: 30 * time.Minute, } - remoteEnodeFlag = cli.StringFlag{ - Name: "remote", - Usage: "Enode of the remote node under test", - EnvVar: "REMOTE_ENODE", + remoteEnodeFlag = &cli.StringFlag{ + Name: "remote", + Usage: "Enode of the remote node under test", + EnvVars: []string{"REMOTE_ENODE"}, } ) +var v4NodeFlags = []cli.Flag{ + bootnodesFlag, + nodekeyFlag, + nodedbFlag, + listenAddrFlag, +} + func discv4Ping(ctx *cli.Context) error { n := getNodeArg(ctx) disc := startV4(ctx) diff --git a/cmd/devp2p/discv5cmd.go b/cmd/devp2p/discv5cmd.go index 873d41e7030cf..298196034b58f 100644 --- a/cmd/devp2p/discv5cmd.go +++ b/cmd/devp2p/discv5cmd.go @@ -23,14 +23,14 @@ import ( "github.com/ethereum/go-ethereum/cmd/devp2p/internal/v5test" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/p2p/discover" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) var ( - discv5Command = cli.Command{ + discv5Command = &cli.Command{ Name: "discv5", Usage: "Node Discovery v5 tools", - Subcommands: []cli.Command{ + Subcommands: []*cli.Command{ discv5PingCommand, discv5ResolveCommand, discv5CrawlCommand, @@ -38,24 +38,24 @@ var ( discv5ListenCommand, }, } - discv5PingCommand = cli.Command{ + discv5PingCommand = &cli.Command{ Name: "ping", Usage: "Sends ping to a node", Action: discv5Ping, } - discv5ResolveCommand = cli.Command{ + discv5ResolveCommand = &cli.Command{ Name: "resolve", Usage: "Finds a node in the DHT", Action: discv5Resolve, Flags: []cli.Flag{bootnodesFlag}, } - discv5CrawlCommand = cli.Command{ + discv5CrawlCommand = &cli.Command{ Name: "crawl", Usage: "Updates a nodes.json file with random nodes found in the DHT", Action: discv5Crawl, Flags: []cli.Flag{bootnodesFlag, crawlTimeoutFlag}, } - discv5TestCommand = cli.Command{ + discv5TestCommand = &cli.Command{ Name: "test", Usage: "Runs protocol tests against a node", Action: discv5Test, @@ -66,7 +66,7 @@ var ( testListen2Flag, }, } - discv5ListenCommand = cli.Command{ + discv5ListenCommand = &cli.Command{ Name: "listen", Usage: "Runs a node", Action: discv5Listen, diff --git a/cmd/devp2p/dns_cloudflare.go b/cmd/devp2p/dns_cloudflare.go index d67aaea1a7fb3..92c6faf272ec4 100644 --- a/cmd/devp2p/dns_cloudflare.go +++ b/cmd/devp2p/dns_cloudflare.go @@ -24,16 +24,16 @@ import ( "github.com/cloudflare/cloudflare-go" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p/dnsdisc" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) var ( - cloudflareTokenFlag = cli.StringFlag{ - Name: "token", - Usage: "CloudFlare API token", - EnvVar: "CLOUDFLARE_API_TOKEN", + cloudflareTokenFlag = &cli.StringFlag{ + Name: "token", + Usage: "CloudFlare API token", + EnvVars: []string{"CLOUDFLARE_API_TOKEN"}, } - cloudflareZoneIDFlag = cli.StringFlag{ + cloudflareZoneIDFlag = &cli.StringFlag{ Name: "zoneid", Usage: "CloudFlare Zone ID (optional)", } @@ -134,7 +134,6 @@ func (c *cloudflareClient) uploadRecords(name string, records map[string]string) ttl := rootTTL if path != name { ttl = treeNodeTTLCloudflare // Max TTL permitted by Cloudflare - } record := cloudflare.DNSRecord{Type: "TXT", Name: path, Content: val, TTL: ttl} _, err = c.CreateDNSRecord(context.Background(), c.zoneID, record) diff --git a/cmd/devp2p/dns_route53.go b/cmd/devp2p/dns_route53.go index 1d4f975dda0b1..4aab0856ff909 100644 --- a/cmd/devp2p/dns_route53.go +++ b/cmd/devp2p/dns_route53.go @@ -32,7 +32,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/route53/types" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p/dnsdisc" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) const ( @@ -45,21 +45,21 @@ const ( ) var ( - route53AccessKeyFlag = cli.StringFlag{ - Name: "access-key-id", - Usage: "AWS Access Key ID", - EnvVar: "AWS_ACCESS_KEY_ID", + route53AccessKeyFlag = &cli.StringFlag{ + Name: "access-key-id", + Usage: "AWS Access Key ID", + EnvVars: []string{"AWS_ACCESS_KEY_ID"}, } - route53AccessSecretFlag = cli.StringFlag{ - Name: "access-key-secret", - Usage: "AWS Access Key Secret", - EnvVar: "AWS_SECRET_ACCESS_KEY", + route53AccessSecretFlag = &cli.StringFlag{ + Name: "access-key-secret", + Usage: "AWS Access Key Secret", + EnvVars: []string{"AWS_SECRET_ACCESS_KEY"}, } - route53ZoneIDFlag = cli.StringFlag{ + route53ZoneIDFlag = &cli.StringFlag{ Name: "zone-id", Usage: "Route53 Zone ID", } - route53RegionFlag = cli.StringFlag{ + route53RegionFlag = &cli.StringFlag{ Name: "aws-region", Usage: "AWS Region", Value: "eu-central-1", diff --git a/cmd/devp2p/dnscmd.go b/cmd/devp2p/dnscmd.go index 21138efdc5ccf..58eb6e8db1c16 100644 --- a/cmd/devp2p/dnscmd.go +++ b/cmd/devp2p/dnscmd.go @@ -29,14 +29,14 @@ import ( "github.com/ethereum/go-ethereum/console/prompt" "github.com/ethereum/go-ethereum/p2p/dnsdisc" "github.com/ethereum/go-ethereum/p2p/enode" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) var ( - dnsCommand = cli.Command{ + dnsCommand = &cli.Command{ Name: "dns", Usage: "DNS Discovery Commands", - Subcommands: []cli.Command{ + Subcommands: []*cli.Command{ dnsSyncCommand, dnsSignCommand, dnsTXTCommand, @@ -45,34 +45,34 @@ var ( dnsRoute53NukeCommand, }, } - dnsSyncCommand = cli.Command{ + dnsSyncCommand = &cli.Command{ Name: "sync", Usage: "Download a DNS discovery tree", ArgsUsage: " [ ]", Action: dnsSync, Flags: []cli.Flag{dnsTimeoutFlag}, } - dnsSignCommand = cli.Command{ + dnsSignCommand = &cli.Command{ Name: "sign", Usage: "Sign a DNS discovery tree", ArgsUsage: " ", Action: dnsSign, Flags: []cli.Flag{dnsDomainFlag, dnsSeqFlag}, } - dnsTXTCommand = cli.Command{ + dnsTXTCommand = &cli.Command{ Name: "to-txt", Usage: "Create a DNS TXT records for a discovery tree", ArgsUsage: " ", Action: dnsToTXT, } - dnsCloudflareCommand = cli.Command{ + dnsCloudflareCommand = &cli.Command{ Name: "to-cloudflare", Usage: "Deploy DNS TXT records to CloudFlare", ArgsUsage: "", Action: dnsToCloudflare, Flags: []cli.Flag{cloudflareTokenFlag, cloudflareZoneIDFlag}, } - dnsRoute53Command = cli.Command{ + dnsRoute53Command = &cli.Command{ Name: "to-route53", Usage: "Deploy DNS TXT records to Amazon Route53", ArgsUsage: "", @@ -84,7 +84,7 @@ var ( route53RegionFlag, }, } - dnsRoute53NukeCommand = cli.Command{ + dnsRoute53NukeCommand = &cli.Command{ Name: "nuke-route53", Usage: "Deletes DNS TXT records of a subdomain on Amazon Route53", ArgsUsage: "", @@ -99,15 +99,15 @@ var ( ) var ( - dnsTimeoutFlag = cli.DurationFlag{ + dnsTimeoutFlag = &cli.DurationFlag{ Name: "timeout", Usage: "Timeout for DNS lookups", } - dnsDomainFlag = cli.StringFlag{ + dnsDomainFlag = &cli.StringFlag{ Name: "domain", Usage: "Domain name of the tree", } - dnsSeqFlag = cli.UintFlag{ + dnsSeqFlag = &cli.UintFlag{ Name: "seq", Usage: "New sequence number of the tree", } diff --git a/cmd/devp2p/enrcmd.go b/cmd/devp2p/enrcmd.go index 2a8f9d508fbe5..2110437103584 100644 --- a/cmd/devp2p/enrcmd.go +++ b/cmd/devp2p/enrcmd.go @@ -30,12 +30,12 @@ import ( "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/p2p/enr" "github.com/ethereum/go-ethereum/rlp" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) -var fileFlag = cli.StringFlag{Name: "file"} +var fileFlag = &cli.StringFlag{Name: "file"} -var enrdumpCommand = cli.Command{ +var enrdumpCommand = &cli.Command{ Name: "enrdump", Usage: "Pretty-prints node records", Action: enrdump, @@ -62,7 +62,7 @@ func enrdump(ctx *cli.Context) error { } source = string(b) } else if ctx.NArg() == 1 { - source = ctx.Args()[0] + source = ctx.Args().First() } else { return fmt.Errorf("need record as argument") } diff --git a/cmd/devp2p/internal/ethtest/chain.go b/cmd/devp2p/internal/ethtest/chain.go index c1d696b40728f..83ceb2a4f2c54 100644 --- a/cmd/devp2p/internal/ethtest/chain.go +++ b/cmd/devp2p/internal/ethtest/chain.go @@ -47,7 +47,7 @@ func (c *Chain) Len() int { // TD calculates the total difficulty of the chain at the // chain head. func (c *Chain) TD() *big.Int { - sum := big.NewInt(0) + sum := new(big.Int) for _, block := range c.blocks[:c.Len()] { sum.Add(sum, block.Difficulty()) } @@ -57,7 +57,7 @@ func (c *Chain) TD() *big.Int { // TotalDifficultyAt calculates the total difficulty of the chain // at the given block height. func (c *Chain) TotalDifficultyAt(height int) *big.Int { - sum := big.NewInt(0) + sum := new(big.Int) if height >= c.Len() { return sum } @@ -96,12 +96,12 @@ func (c *Chain) Head() *types.Block { return c.blocks[c.Len()-1] } -func (c *Chain) GetHeaders(req GetBlockHeaders) (BlockHeaders, error) { +func (c *Chain) GetHeaders(req *GetBlockHeaders) ([]*types.Header, error) { if req.Amount < 1 { return nil, fmt.Errorf("no block headers requested") } - headers := make(BlockHeaders, req.Amount) + headers := make([]*types.Header, req.Amount) var blockNumber uint64 // range over blocks to check if our chain has the requested header @@ -119,7 +119,6 @@ func (c *Chain) GetHeaders(req GetBlockHeaders) (BlockHeaders, error) { for i := 1; i < int(req.Amount); i++ { blockNumber -= (1 - req.Skip) headers[i] = c.blocks[blockNumber].Header() - } return headers, nil @@ -140,7 +139,7 @@ func loadChain(chainfile string, genesis string) (*Chain, error) { if err != nil { return nil, err } - gblock := gen.ToBlock(nil) + gblock := gen.ToBlock() blocks, err := blocksFromFile(chainfile, gblock) if err != nil { diff --git a/cmd/devp2p/internal/ethtest/chain_test.go b/cmd/devp2p/internal/ethtest/chain_test.go index 0f232b150611a..67221923a6844 100644 --- a/cmd/devp2p/internal/ethtest/chain_test.go +++ b/cmd/devp2p/internal/ethtest/chain_test.go @@ -21,6 +21,7 @@ import ( "strconv" "testing" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth/protocols/eth" "github.com/ethereum/go-ethereum/p2p" "github.com/stretchr/testify/assert" @@ -140,18 +141,18 @@ func TestChain_GetHeaders(t *testing.T) { var tests = []struct { req GetBlockHeaders - expected BlockHeaders + expected []*types.Header }{ { req: GetBlockHeaders{ - Origin: eth.HashOrNumber{ - Number: uint64(2), + GetBlockHeadersPacket: ð.GetBlockHeadersPacket{ + Origin: eth.HashOrNumber{Number: uint64(2)}, + Amount: uint64(5), + Skip: 1, + Reverse: false, }, - Amount: uint64(5), - Skip: 1, - Reverse: false, }, - expected: BlockHeaders{ + expected: []*types.Header{ chain.blocks[2].Header(), chain.blocks[4].Header(), chain.blocks[6].Header(), @@ -161,14 +162,14 @@ func TestChain_GetHeaders(t *testing.T) { }, { req: GetBlockHeaders{ - Origin: eth.HashOrNumber{ - Number: uint64(chain.Len() - 1), + GetBlockHeadersPacket: ð.GetBlockHeadersPacket{ + Origin: eth.HashOrNumber{Number: uint64(chain.Len() - 1)}, + Amount: uint64(3), + Skip: 0, + Reverse: true, }, - Amount: uint64(3), - Skip: 0, - Reverse: true, }, - expected: BlockHeaders{ + expected: []*types.Header{ chain.blocks[chain.Len()-1].Header(), chain.blocks[chain.Len()-2].Header(), chain.blocks[chain.Len()-3].Header(), @@ -176,14 +177,14 @@ func TestChain_GetHeaders(t *testing.T) { }, { req: GetBlockHeaders{ - Origin: eth.HashOrNumber{ - Hash: chain.Head().Hash(), + GetBlockHeadersPacket: ð.GetBlockHeadersPacket{ + Origin: eth.HashOrNumber{Hash: chain.Head().Hash()}, + Amount: uint64(1), + Skip: 0, + Reverse: false, }, - Amount: uint64(1), - Skip: 0, - Reverse: false, }, - expected: BlockHeaders{ + expected: []*types.Header{ chain.Head().Header(), }, }, @@ -191,7 +192,7 @@ func TestChain_GetHeaders(t *testing.T) { for i, tt := range tests { t.Run(strconv.Itoa(i), func(t *testing.T) { - headers, err := chain.GetHeaders(tt.req) + headers, err := chain.GetHeaders(&tt.req) if err != nil { t.Fatal(err) } diff --git a/cmd/devp2p/internal/ethtest/helpers.go b/cmd/devp2p/internal/ethtest/helpers.go index df754d6ce61dc..eeeb4f93cabfd 100644 --- a/cmd/devp2p/internal/ethtest/helpers.go +++ b/cmd/devp2p/internal/ethtest/helpers.go @@ -43,21 +43,6 @@ var ( timeout = 20 * time.Second ) -// Is_66 checks if the node supports the eth66 protocol version, -// and if not, exists the test suite -func (s *Suite) Is_66(t *utesting.T) { - conn, err := s.dial66() - if err != nil { - t.Fatalf("dial failed: %v", err) - } - if err := conn.handshake(); err != nil { - t.Fatalf("handshake failed: %v", err) - } - if conn.negotiatedProtoVersion < 66 { - t.Fail() - } -} - // dial attempts to dial the given node and perform a handshake, // returning the created Conn if successful. func (s *Suite) dial() (*Conn, error) { @@ -76,31 +61,16 @@ func (s *Suite) dial() (*Conn, error) { } // set default p2p capabilities conn.caps = []p2p.Cap{ - {Name: "eth", Version: 64}, - {Name: "eth", Version: 65}, + {Name: "eth", Version: 66}, + {Name: "eth", Version: 67}, } - conn.ourHighestProtoVersion = 65 + conn.ourHighestProtoVersion = 67 return &conn, nil } -// dial66 attempts to dial the given node and perform a handshake, -// returning the created Conn with additional eth66 capabilities if -// successful -func (s *Suite) dial66() (*Conn, error) { - conn, err := s.dial() - if err != nil { - return nil, fmt.Errorf("dial failed: %v", err) - } - conn.caps = append(conn.caps, p2p.Cap{Name: "eth", Version: 66}) - conn.ourHighestProtoVersion = 66 - return conn, nil -} - -// dial66 attempts to dial the given node and perform a handshake, -// returning the created Conn with additional snap/1 capabilities if -// successful. +// dialSnap creates a connection with snap/1 capability. func (s *Suite) dialSnap() (*Conn, error) { - conn, err := s.dial66() + conn, err := s.dial() if err != nil { return nil, fmt.Errorf("dial failed: %v", err) } @@ -235,117 +205,68 @@ loop: // createSendAndRecvConns creates two connections, one for sending messages to the // node, and one for receiving messages from the node. -func (s *Suite) createSendAndRecvConns(isEth66 bool) (*Conn, *Conn, error) { - var ( - sendConn *Conn - recvConn *Conn - err error - ) - if isEth66 { - sendConn, err = s.dial66() - if err != nil { - return nil, nil, fmt.Errorf("dial failed: %v", err) - } - recvConn, err = s.dial66() - if err != nil { - sendConn.Close() - return nil, nil, fmt.Errorf("dial failed: %v", err) - } - } else { - sendConn, err = s.dial() - if err != nil { - return nil, nil, fmt.Errorf("dial failed: %v", err) - } - recvConn, err = s.dial() - if err != nil { - sendConn.Close() - return nil, nil, fmt.Errorf("dial failed: %v", err) - } +func (s *Suite) createSendAndRecvConns() (*Conn, *Conn, error) { + sendConn, err := s.dial() + if err != nil { + return nil, nil, fmt.Errorf("dial failed: %v", err) } - return sendConn, recvConn, nil -} - -func (c *Conn) readAndServe(chain *Chain, timeout time.Duration) Message { - if c.negotiatedProtoVersion == 66 { - _, msg := c.readAndServe66(chain, timeout) - return msg + recvConn, err := s.dial() + if err != nil { + sendConn.Close() + return nil, nil, fmt.Errorf("dial failed: %v", err) } - return c.readAndServe65(chain, timeout) + return sendConn, recvConn, nil } // readAndServe serves GetBlockHeaders requests while waiting // on another message from the node. -func (c *Conn) readAndServe65(chain *Chain, timeout time.Duration) Message { - start := time.Now() - for time.Since(start) < timeout { - c.SetReadDeadline(time.Now().Add(5 * time.Second)) - switch msg := c.Read().(type) { - case *Ping: - c.Write(&Pong{}) - case *GetBlockHeaders: - req := *msg - headers, err := chain.GetHeaders(req) - if err != nil { - return errorf("could not get headers for inbound header request: %v", err) - } - if err := c.Write(headers); err != nil { - return errorf("could not write to connection: %v", err) - } - default: - return msg - } - } - return errorf("no message received within %v", timeout) -} - -// readAndServe66 serves eth66 GetBlockHeaders requests while waiting -// on another message from the node. -func (c *Conn) readAndServe66(chain *Chain, timeout time.Duration) (uint64, Message) { +func (c *Conn) readAndServe(chain *Chain, timeout time.Duration) Message { start := time.Now() for time.Since(start) < timeout { c.SetReadDeadline(time.Now().Add(10 * time.Second)) - reqID, msg := c.Read66() - + msg := c.Read() switch msg := msg.(type) { case *Ping: c.Write(&Pong{}) - case GetBlockHeaders: + case *GetBlockHeaders: headers, err := chain.GetHeaders(msg) if err != nil { - return 0, errorf("could not get headers for inbound header request: %v", err) + return errorf("could not get headers for inbound header request: %v", err) } - resp := ð.BlockHeadersPacket66{ - RequestId: reqID, + resp := &BlockHeaders{ + RequestId: msg.ReqID(), BlockHeadersPacket: eth.BlockHeadersPacket(headers), } - if err := c.Write66(resp, BlockHeaders{}.Code()); err != nil { - return 0, errorf("could not write to connection: %v", err) + if err := c.Write(resp); err != nil { + return errorf("could not write to connection: %v", err) } default: - return reqID, msg + return msg } } - return 0, errorf("no message received within %v", timeout) + return errorf("no message received within %v", timeout) } // headersRequest executes the given `GetBlockHeaders` request. -func (c *Conn) headersRequest(request *GetBlockHeaders, chain *Chain, isEth66 bool, reqID uint64) (BlockHeaders, error) { +func (c *Conn) headersRequest(request *GetBlockHeaders, chain *Chain, reqID uint64) ([]*types.Header, error) { defer c.SetReadDeadline(time.Time{}) c.SetReadDeadline(time.Now().Add(20 * time.Second)) - // if on eth66 connection, perform eth66 GetBlockHeaders request - if isEth66 { - return getBlockHeaders66(chain, c, request, reqID) - } + + // write request + request.RequestId = reqID if err := c.Write(request); err != nil { - return nil, err + return nil, fmt.Errorf("could not write to connection: %v", err) } - switch msg := c.readAndServe(chain, timeout).(type) { - case *BlockHeaders: - return *msg, nil - default: - return nil, fmt.Errorf("invalid message: %s", pretty.Sdump(msg)) + + // wait for response + msg := c.waitForResponse(chain, timeout, request.RequestId) + resp, ok := msg.(*BlockHeaders) + if !ok { + return nil, fmt.Errorf("unexpected message received: %s", pretty.Sdump(msg)) } + headers := []*types.Header(resp.BlockHeadersPacket) + return headers, nil } func (c *Conn) snapRequest(msg Message, id uint64, chain *Chain) (Message, error) { @@ -357,28 +278,8 @@ func (c *Conn) snapRequest(msg Message, id uint64, chain *Chain) (Message, error return c.ReadSnap(id) } -// getBlockHeaders66 executes the given `GetBlockHeaders` request over the eth66 protocol. -func getBlockHeaders66(chain *Chain, conn *Conn, request *GetBlockHeaders, id uint64) (BlockHeaders, error) { - // write request - packet := eth.GetBlockHeadersPacket(*request) - req := ð.GetBlockHeadersPacket66{ - RequestId: id, - GetBlockHeadersPacket: &packet, - } - if err := conn.Write66(req, GetBlockHeaders{}.Code()); err != nil { - return nil, fmt.Errorf("could not write to connection: %v", err) - } - // wait for response - msg := conn.waitForResponse(chain, timeout, req.RequestId) - headers, ok := msg.(BlockHeaders) - if !ok { - return nil, fmt.Errorf("unexpected message received: %s", pretty.Sdump(msg)) - } - return headers, nil -} - // headersMatch returns whether the received headers match the given request -func headersMatch(expected BlockHeaders, headers BlockHeaders) bool { +func headersMatch(expected []*types.Header, headers []*types.Header) bool { return reflect.DeepEqual(expected, headers) } @@ -386,8 +287,8 @@ func headersMatch(expected BlockHeaders, headers BlockHeaders) bool { // request ID is received. func (c *Conn) waitForResponse(chain *Chain, timeout time.Duration, requestID uint64) Message { for { - id, msg := c.readAndServe66(chain, timeout) - if id == requestID { + msg := c.readAndServe(chain, timeout) + if msg.ReqID() == requestID { return msg } } @@ -395,9 +296,9 @@ func (c *Conn) waitForResponse(chain *Chain, timeout time.Duration, requestID ui // sendNextBlock broadcasts the next block in the chain and waits // for the node to propagate the block and import it into its chain. -func (s *Suite) sendNextBlock(isEth66 bool) error { +func (s *Suite) sendNextBlock() error { // set up sending and receiving connections - sendConn, recvConn, err := s.createSendAndRecvConns(isEth66) + sendConn, recvConn, err := s.createSendAndRecvConns() if err != nil { return err } @@ -420,7 +321,7 @@ func (s *Suite) sendNextBlock(isEth66 bool) error { return fmt.Errorf("failed to announce block: %v", err) } // wait for client to update its chain - if err = s.waitForBlockImport(recvConn, nextBlock, isEth66); err != nil { + if err = s.waitForBlockImport(recvConn, nextBlock); err != nil { return fmt.Errorf("failed to receive confirmation of block import: %v", err) } // update test suite chain @@ -465,29 +366,22 @@ func (s *Suite) waitAnnounce(conn *Conn, blockAnnouncement *NewBlock) error { } } -func (s *Suite) waitForBlockImport(conn *Conn, block *types.Block, isEth66 bool) error { +func (s *Suite) waitForBlockImport(conn *Conn, block *types.Block) error { defer conn.SetReadDeadline(time.Time{}) conn.SetReadDeadline(time.Now().Add(20 * time.Second)) // create request req := &GetBlockHeaders{ - Origin: eth.HashOrNumber{ - Hash: block.Hash(), + GetBlockHeadersPacket: ð.GetBlockHeadersPacket{ + Origin: eth.HashOrNumber{Hash: block.Hash()}, + Amount: 1, }, - Amount: 1, } + // loop until BlockHeaders response contains desired block, confirming the // node imported the block for { - var ( - headers BlockHeaders - err error - ) - if isEth66 { - requestID := uint64(54) - headers, err = conn.headersRequest(req, s.chain, eth66, requestID) - } else { - headers, err = conn.headersRequest(req, s.chain, eth65, 0) - } + requestID := uint64(54) + headers, err := conn.headersRequest(req, s.chain, requestID) if err != nil { return fmt.Errorf("GetBlockHeader request failed: %v", err) } @@ -503,8 +397,8 @@ func (s *Suite) waitForBlockImport(conn *Conn, block *types.Block, isEth66 bool) } } -func (s *Suite) oldAnnounce(isEth66 bool) error { - sendConn, receiveConn, err := s.createSendAndRecvConns(isEth66) +func (s *Suite) oldAnnounce() error { + sendConn, receiveConn, err := s.createSendAndRecvConns() if err != nil { return err } @@ -550,23 +444,13 @@ func (s *Suite) oldAnnounce(isEth66 bool) error { return nil } -func (s *Suite) maliciousHandshakes(t *utesting.T, isEth66 bool) error { - var ( - conn *Conn - err error - ) - if isEth66 { - conn, err = s.dial66() - if err != nil { - return fmt.Errorf("dial failed: %v", err) - } - } else { - conn, err = s.dial() - if err != nil { - return fmt.Errorf("dial failed: %v", err) - } +func (s *Suite) maliciousHandshakes(t *utesting.T) error { + conn, err := s.dial() + if err != nil { + return fmt.Errorf("dial failed: %v", err) } defer conn.Close() + // write hello to client pub0 := crypto.FromECDSAPub(&conn.ourKey.PublicKey)[1:] handshakes := []*Hello{ @@ -627,16 +511,9 @@ func (s *Suite) maliciousHandshakes(t *utesting.T, isEth66 bool) error { } } // dial for the next round - if isEth66 { - conn, err = s.dial66() - if err != nil { - return fmt.Errorf("dial failed: %v", err) - } - } else { - conn, err = s.dial() - if err != nil { - return fmt.Errorf("dial failed: %v", err) - } + conn, err = s.dial() + if err != nil { + return fmt.Errorf("dial failed: %v", err) } } return nil @@ -654,6 +531,7 @@ func (s *Suite) maliciousStatus(conn *Conn) error { Genesis: s.chain.blocks[0].Hash(), ForkID: s.chain.ForkID(), } + // get status msg, err := conn.statusExchange(s.chain, status) if err != nil { @@ -664,6 +542,7 @@ func (s *Suite) maliciousStatus(conn *Conn) error { default: return fmt.Errorf("expected status, got: %#v ", msg) } + // wait for disconnect switch msg := conn.readAndServe(s.chain, timeout).(type) { case *Disconnect: @@ -675,9 +554,9 @@ func (s *Suite) maliciousStatus(conn *Conn) error { } } -func (s *Suite) hashAnnounce(isEth66 bool) error { +func (s *Suite) hashAnnounce() error { // create connections - sendConn, recvConn, err := s.createSendAndRecvConns(isEth66) + sendConn, recvConn, err := s.createSendAndRecvConns() if err != nil { return fmt.Errorf("failed to create connections: %v", err) } @@ -689,6 +568,7 @@ func (s *Suite) hashAnnounce(isEth66 bool) error { if err := recvConn.peer(s.chain, nil); err != nil { return fmt.Errorf("peering failed: %v", err) } + // create NewBlockHashes announcement type anno struct { Hash common.Hash // Hash of one particular block being announced @@ -700,56 +580,29 @@ func (s *Suite) hashAnnounce(isEth66 bool) error { if err := sendConn.Write(newBlockHash); err != nil { return fmt.Errorf("failed to write to connection: %v", err) } + // Announcement sent, now wait for a header request - var ( - id uint64 - msg Message - blockHeaderReq GetBlockHeaders - ) - if isEth66 { - id, msg = sendConn.Read66() - switch msg := msg.(type) { - case GetBlockHeaders: - blockHeaderReq = msg - default: - return fmt.Errorf("unexpected %s", pretty.Sdump(msg)) - } - if blockHeaderReq.Amount != 1 { - return fmt.Errorf("unexpected number of block headers requested: %v", blockHeaderReq.Amount) - } - if blockHeaderReq.Origin.Hash != announcement.Hash { - return fmt.Errorf("unexpected block header requested. Announced:\n %v\n Remote request:\n%v", - pretty.Sdump(announcement), - pretty.Sdump(blockHeaderReq)) - } - if err := sendConn.Write66(ð.BlockHeadersPacket66{ - RequestId: id, - BlockHeadersPacket: eth.BlockHeadersPacket{ - nextBlock.Header(), - }, - }, BlockHeaders{}.Code()); err != nil { - return fmt.Errorf("failed to write to connection: %v", err) - } - } else { - msg = sendConn.Read() - switch msg := msg.(type) { - case *GetBlockHeaders: - blockHeaderReq = *msg - default: - return fmt.Errorf("unexpected %s", pretty.Sdump(msg)) - } - if blockHeaderReq.Amount != 1 { - return fmt.Errorf("unexpected number of block headers requested: %v", blockHeaderReq.Amount) - } - if blockHeaderReq.Origin.Hash != announcement.Hash { - return fmt.Errorf("unexpected block header requested. Announced:\n %v\n Remote request:\n%v", - pretty.Sdump(announcement), - pretty.Sdump(blockHeaderReq)) - } - if err := sendConn.Write(&BlockHeaders{nextBlock.Header()}); err != nil { - return fmt.Errorf("failed to write to connection: %v", err) - } + msg := sendConn.Read() + blockHeaderReq, ok := msg.(*GetBlockHeaders) + if !ok { + return fmt.Errorf("unexpected %s", pretty.Sdump(msg)) } + if blockHeaderReq.Amount != 1 { + return fmt.Errorf("unexpected number of block headers requested: %v", blockHeaderReq.Amount) + } + if blockHeaderReq.Origin.Hash != announcement.Hash { + return fmt.Errorf("unexpected block header requested. Announced:\n %v\n Remote request:\n%v", + pretty.Sdump(announcement), + pretty.Sdump(blockHeaderReq)) + } + err = sendConn.Write(&BlockHeaders{ + RequestId: blockHeaderReq.ReqID(), + BlockHeadersPacket: eth.BlockHeadersPacket{nextBlock.Header()}, + }) + if err != nil { + return fmt.Errorf("failed to write to connection: %v", err) + } + // wait for block announcement msg = recvConn.readAndServe(s.chain, timeout) switch msg := msg.(type) { @@ -762,6 +615,7 @@ func (s *Suite) hashAnnounce(isEth66 bool) error { return fmt.Errorf("unexpected block hash announcement, wanted %v, got %v", nextBlock.Hash(), hashes[0].Hash) } + case *NewBlock: // node should only propagate NewBlock without having requested the body if the body is empty nextBlockBody := nextBlock.Body() @@ -780,7 +634,7 @@ func (s *Suite) hashAnnounce(isEth66 bool) error { return fmt.Errorf("unexpected: %s", pretty.Sdump(msg)) } // confirm node imported block - if err := s.waitForBlockImport(recvConn, nextBlock, isEth66); err != nil { + if err := s.waitForBlockImport(recvConn, nextBlock); err != nil { return fmt.Errorf("error waiting for node to import new block: %v", err) } // update the chain diff --git a/cmd/devp2p/internal/ethtest/snap.go b/cmd/devp2p/internal/ethtest/snap.go index 2bfd29c75abf1..032afeafcdad6 100644 --- a/cmd/devp2p/internal/ethtest/snap.go +++ b/cmd/devp2p/internal/ethtest/snap.go @@ -104,6 +104,7 @@ func (s *Suite) TestSnapGetAccountRange(t *utesting.T) { // Max bytes: 0. Expect to deliver one account. {0, root, zero, ffHash, 1, firstKey, firstKey}, } { + tc := tc if err := s.snapGetAccountRange(t, &tc); err != nil { t.Errorf("test %d \n root: %x\n range: %#x - %#x\n bytes: %d\nfailed: %v", i, tc.root, tc.origin, tc.limit, tc.nBytes, err) } @@ -194,6 +195,7 @@ func (s *Suite) TestSnapGetStorageRanges(t *utesting.T) { expSlots: 2, }, } { + tc := tc if err := s.snapGetStorageRanges(t, &tc); err != nil { t.Errorf("test %d \n root: %x\n range: %#x - %#x\n bytes: %d\n #accounts: %d\nfailed: %v", i, tc.root, tc.origin, tc.limit, tc.nBytes, len(tc.accounts), err) @@ -291,6 +293,7 @@ func (s *Suite) TestSnapGetByteCodes(t *utesting.T) { expHashes: 4, }, } { + tc := tc if err := s.snapGetByteCodes(t, &tc); err != nil { t.Errorf("test %d \n bytes: %d\n #hashes: %d\nfailed: %v", i, tc.nBytes, len(tc.hashes), err) } @@ -347,7 +350,6 @@ func hexToCompact(hex []byte) []byte { // TestSnapTrieNodes various forms of GetTrieNodes requests. func (s *Suite) TestSnapTrieNodes(t *utesting.T) { - key := common.FromHex("0x00bf49f440a1cd0527e4d06e2765654c0f56452257516d793a9b8d604dcfdf2a") // helper function to iterate the key, and generate the compact-encoded // trie paths along the way. @@ -436,6 +438,7 @@ func (s *Suite) TestSnapTrieNodes(t *utesting.T) { }, }, } { + tc := tc if err := s.snapGetTrieNodes(t, &tc); err != nil { t.Errorf("test %d \n #hashes %x\n root: %#x\n bytes: %d\nfailed: %v", i, len(tc.expHashes), tc.root, tc.nBytes, err) } @@ -492,10 +495,10 @@ func (s *Suite) snapGetAccountRange(t *utesting.T, tc *accRangeTest) error { } if len(hashes) > 0 { if exp, got := tc.expFirst, res.Accounts[0].Hash; exp != got { - return fmt.Errorf("expected first account 0x%x, got 0x%x", exp, got) + return fmt.Errorf("expected first account %#x, got %#x", exp, got) } if exp, got := tc.expLast, res.Accounts[len(res.Accounts)-1].Hash; exp != got { - return fmt.Errorf("expected last account 0x%x, got 0x%x", exp, got) + return fmt.Errorf("expected last account %#x, got %#x", exp, got) } } // Reconstruct a partial trie from the response and verify it diff --git a/cmd/devp2p/internal/ethtest/snapTypes.go b/cmd/devp2p/internal/ethtest/snapTypes.go index e18cd5925cbb6..6bcaa9291ab22 100644 --- a/cmd/devp2p/internal/ethtest/snapTypes.go +++ b/cmd/devp2p/internal/ethtest/snapTypes.go @@ -21,32 +21,40 @@ import "github.com/ethereum/go-ethereum/eth/protocols/snap" // GetAccountRange represents an account range query. type GetAccountRange snap.GetAccountRangePacket -func (g GetAccountRange) Code() int { return 33 } +func (msg GetAccountRange) Code() int { return 33 } +func (msg GetAccountRange) ReqID() uint64 { return msg.ID } type AccountRange snap.AccountRangePacket -func (g AccountRange) Code() int { return 34 } +func (msg AccountRange) Code() int { return 34 } +func (msg AccountRange) ReqID() uint64 { return msg.ID } type GetStorageRanges snap.GetStorageRangesPacket -func (g GetStorageRanges) Code() int { return 35 } +func (msg GetStorageRanges) Code() int { return 35 } +func (msg GetStorageRanges) ReqID() uint64 { return msg.ID } type StorageRanges snap.StorageRangesPacket -func (g StorageRanges) Code() int { return 36 } +func (msg StorageRanges) Code() int { return 36 } +func (msg StorageRanges) ReqID() uint64 { return msg.ID } type GetByteCodes snap.GetByteCodesPacket -func (g GetByteCodes) Code() int { return 37 } +func (msg GetByteCodes) Code() int { return 37 } +func (msg GetByteCodes) ReqID() uint64 { return msg.ID } type ByteCodes snap.ByteCodesPacket -func (g ByteCodes) Code() int { return 38 } +func (msg ByteCodes) Code() int { return 38 } +func (msg ByteCodes) ReqID() uint64 { return msg.ID } type GetTrieNodes snap.GetTrieNodesPacket -func (g GetTrieNodes) Code() int { return 39 } +func (msg GetTrieNodes) Code() int { return 39 } +func (msg GetTrieNodes) ReqID() uint64 { return msg.ID } type TrieNodes snap.TrieNodesPacket -func (g TrieNodes) Code() int { return 40 } +func (msg TrieNodes) Code() int { return 40 } +func (msg TrieNodes) ReqID() uint64 { return msg.ID } diff --git a/cmd/devp2p/internal/ethtest/suite.go b/cmd/devp2p/internal/ethtest/suite.go index 4ddd65b95865b..7059b4ba738cd 100644 --- a/cmd/devp2p/internal/ethtest/suite.go +++ b/cmd/devp2p/internal/ethtest/suite.go @@ -49,79 +49,30 @@ func NewSuite(dest *enode.Node, chainfile string, genesisfile string) (*Suite, e }, nil } -func (s *Suite) AllEthTests() []utesting.Test { +func (s *Suite) EthTests() []utesting.Test { return []utesting.Test{ // status - {Name: "TestStatus65", Fn: s.TestStatus65}, - {Name: "TestStatus66", Fn: s.TestStatus66}, + {Name: "TestStatus", Fn: s.TestStatus}, // get block headers - {Name: "TestGetBlockHeaders65", Fn: s.TestGetBlockHeaders65}, - {Name: "TestGetBlockHeaders66", Fn: s.TestGetBlockHeaders66}, - {Name: "TestSimultaneousRequests66", Fn: s.TestSimultaneousRequests66}, - {Name: "TestSameRequestID66", Fn: s.TestSameRequestID66}, - {Name: "TestZeroRequestID66", Fn: s.TestZeroRequestID66}, + {Name: "TestGetBlockHeaders", Fn: s.TestGetBlockHeaders}, + {Name: "TestSimultaneousRequests", Fn: s.TestSimultaneousRequests}, + {Name: "TestSameRequestID", Fn: s.TestSameRequestID}, + {Name: "TestZeroRequestID", Fn: s.TestZeroRequestID}, // get block bodies - {Name: "TestGetBlockBodies65", Fn: s.TestGetBlockBodies65}, - {Name: "TestGetBlockBodies66", Fn: s.TestGetBlockBodies66}, + {Name: "TestGetBlockBodies", Fn: s.TestGetBlockBodies}, // broadcast - {Name: "TestBroadcast65", Fn: s.TestBroadcast65}, - {Name: "TestBroadcast66", Fn: s.TestBroadcast66}, - {Name: "TestLargeAnnounce65", Fn: s.TestLargeAnnounce65}, - {Name: "TestLargeAnnounce66", Fn: s.TestLargeAnnounce66}, - {Name: "TestOldAnnounce65", Fn: s.TestOldAnnounce65}, - {Name: "TestOldAnnounce66", Fn: s.TestOldAnnounce66}, - {Name: "TestBlockHashAnnounce65", Fn: s.TestBlockHashAnnounce65}, - {Name: "TestBlockHashAnnounce66", Fn: s.TestBlockHashAnnounce66}, + {Name: "TestBroadcast", Fn: s.TestBroadcast}, + {Name: "TestLargeAnnounce", Fn: s.TestLargeAnnounce}, + {Name: "TestOldAnnounce", Fn: s.TestOldAnnounce}, + {Name: "TestBlockHashAnnounce", Fn: s.TestBlockHashAnnounce}, // malicious handshakes + status - {Name: "TestMaliciousHandshake65", Fn: s.TestMaliciousHandshake65}, - {Name: "TestMaliciousStatus65", Fn: s.TestMaliciousStatus65}, - {Name: "TestMaliciousHandshake66", Fn: s.TestMaliciousHandshake66}, - {Name: "TestMaliciousStatus66", Fn: s.TestMaliciousStatus66}, + {Name: "TestMaliciousHandshake", Fn: s.TestMaliciousHandshake}, + {Name: "TestMaliciousStatus", Fn: s.TestMaliciousStatus}, // test transactions - {Name: "TestTransaction65", Fn: s.TestTransaction65}, - {Name: "TestTransaction66", Fn: s.TestTransaction66}, - {Name: "TestMaliciousTx65", Fn: s.TestMaliciousTx65}, - {Name: "TestMaliciousTx66", Fn: s.TestMaliciousTx66}, - {Name: "TestLargeTxRequest66", Fn: s.TestLargeTxRequest66}, - {Name: "TestNewPooledTxs66", Fn: s.TestNewPooledTxs66}, - } -} - -func (s *Suite) EthTests() []utesting.Test { - return []utesting.Test{ - {Name: "TestStatus65", Fn: s.TestStatus65}, - {Name: "TestGetBlockHeaders65", Fn: s.TestGetBlockHeaders65}, - {Name: "TestGetBlockBodies65", Fn: s.TestGetBlockBodies65}, - {Name: "TestBroadcast65", Fn: s.TestBroadcast65}, - {Name: "TestLargeAnnounce65", Fn: s.TestLargeAnnounce65}, - {Name: "TestOldAnnounce65", Fn: s.TestOldAnnounce65}, - {Name: "TestBlockHashAnnounce65", Fn: s.TestBlockHashAnnounce65}, - {Name: "TestMaliciousHandshake65", Fn: s.TestMaliciousHandshake65}, - {Name: "TestMaliciousStatus65", Fn: s.TestMaliciousStatus65}, - {Name: "TestTransaction65", Fn: s.TestTransaction65}, - {Name: "TestMaliciousTx65", Fn: s.TestMaliciousTx65}, - } -} - -func (s *Suite) Eth66Tests() []utesting.Test { - return []utesting.Test{ - // only proceed with eth66 test suite if node supports eth 66 protocol - {Name: "TestStatus66", Fn: s.TestStatus66}, - {Name: "TestGetBlockHeaders66", Fn: s.TestGetBlockHeaders66}, - {Name: "TestSimultaneousRequests66", Fn: s.TestSimultaneousRequests66}, - {Name: "TestSameRequestID66", Fn: s.TestSameRequestID66}, - {Name: "TestZeroRequestID66", Fn: s.TestZeroRequestID66}, - {Name: "TestGetBlockBodies66", Fn: s.TestGetBlockBodies66}, - {Name: "TestBroadcast66", Fn: s.TestBroadcast66}, - {Name: "TestLargeAnnounce66", Fn: s.TestLargeAnnounce66}, - {Name: "TestOldAnnounce66", Fn: s.TestOldAnnounce66}, - {Name: "TestBlockHashAnnounce66", Fn: s.TestBlockHashAnnounce66}, - {Name: "TestMaliciousHandshake66", Fn: s.TestMaliciousHandshake66}, - {Name: "TestMaliciousStatus66", Fn: s.TestMaliciousStatus66}, - {Name: "TestTransaction66", Fn: s.TestTransaction66}, - {Name: "TestMaliciousTx66", Fn: s.TestMaliciousTx66}, - {Name: "TestLargeTxRequest66", Fn: s.TestLargeTxRequest66}, - {Name: "TestNewPooledTxs66", Fn: s.TestNewPooledTxs66}, + {Name: "TestTransaction", Fn: s.TestTransaction}, + {Name: "TestMaliciousTx", Fn: s.TestMaliciousTx}, + {Name: "TestLargeTxRequest", Fn: s.TestLargeTxRequest}, + {Name: "TestNewPooledTxs", Fn: s.TestNewPooledTxs}, } } @@ -135,14 +86,9 @@ func (s *Suite) SnapTests() []utesting.Test { } } -var ( - eth66 = true // indicates whether suite should negotiate eth66 connection - eth65 = false // indicates whether suite should negotiate eth65 connection or below. -) - -// TestStatus65 attempts to connect to the given node and exchange -// a status message with it. -func (s *Suite) TestStatus65(t *utesting.T) { +// TestStatus attempts to connect to the given node and exchange +// a status message with it on the eth protocol. +func (s *Suite) TestStatus(t *utesting.T) { conn, err := s.dial() if err != nil { t.Fatalf("dial failed: %v", err) @@ -153,79 +99,32 @@ func (s *Suite) TestStatus65(t *utesting.T) { } } -// TestStatus66 attempts to connect to the given node and exchange -// a status message with it on the eth66 protocol. -func (s *Suite) TestStatus66(t *utesting.T) { - conn, err := s.dial66() - if err != nil { - t.Fatalf("dial failed: %v", err) - } - defer conn.Close() - if err := conn.peer(s.chain, nil); err != nil { - t.Fatalf("peering failed: %v", err) - } -} - -// TestGetBlockHeaders65 tests whether the given node can respond to -// a `GetBlockHeaders` request accurately. -func (s *Suite) TestGetBlockHeaders65(t *utesting.T) { +// TestGetBlockHeaders tests whether the given node can respond to +// an eth `GetBlockHeaders` request and that the response is accurate. +func (s *Suite) TestGetBlockHeaders(t *utesting.T) { conn, err := s.dial() if err != nil { t.Fatalf("dial failed: %v", err) } defer conn.Close() - if err := conn.peer(s.chain, nil); err != nil { - t.Fatalf("handshake(s) failed: %v", err) - } - // write request - req := &GetBlockHeaders{ - Origin: eth.HashOrNumber{ - Hash: s.chain.blocks[1].Hash(), - }, - Amount: 2, - Skip: 1, - Reverse: false, - } - headers, err := conn.headersRequest(req, s.chain, eth65, 0) - if err != nil { - t.Fatalf("GetBlockHeaders request failed: %v", err) - } - // check for correct headers - expected, err := s.chain.GetHeaders(*req) - if err != nil { - t.Fatalf("failed to get headers for given request: %v", err) - } - if !headersMatch(expected, headers) { - t.Fatalf("header mismatch: \nexpected %v \ngot %v", expected, headers) - } -} - -// TestGetBlockHeaders66 tests whether the given node can respond to -// an eth66 `GetBlockHeaders` request and that the response is accurate. -func (s *Suite) TestGetBlockHeaders66(t *utesting.T) { - conn, err := s.dial66() - if err != nil { - t.Fatalf("dial failed: %v", err) - } - defer conn.Close() if err = conn.peer(s.chain, nil); err != nil { t.Fatalf("peering failed: %v", err) } // write request req := &GetBlockHeaders{ - Origin: eth.HashOrNumber{ - Hash: s.chain.blocks[1].Hash(), + GetBlockHeadersPacket: ð.GetBlockHeadersPacket{ + Origin: eth.HashOrNumber{Hash: s.chain.blocks[1].Hash()}, + Amount: 2, + Skip: 1, + Reverse: false, }, - Amount: 2, - Skip: 1, - Reverse: false, } - headers, err := conn.headersRequest(req, s.chain, eth66, 33) + headers, err := conn.headersRequest(req, s.chain, 33) if err != nil { t.Fatalf("could not get block headers: %v", err) } // check for correct headers - expected, err := s.chain.GetHeaders(*req) + expected, err := s.chain.GetHeaders(req) if err != nil { t.Fatalf("failed to get headers for given request: %v", err) } @@ -234,12 +133,12 @@ func (s *Suite) TestGetBlockHeaders66(t *utesting.T) { } } -// TestSimultaneousRequests66 sends two simultaneous `GetBlockHeader` requests from +// TestSimultaneousRequests sends two simultaneous `GetBlockHeader` requests from // the same connection with different request IDs and checks to make sure the node // responds with the correct headers per request. -func (s *Suite) TestSimultaneousRequests66(t *utesting.T) { +func (s *Suite) TestSimultaneousRequests(t *utesting.T) { // create a connection - conn, err := s.dial66() + conn, err := s.dial() if err != nil { t.Fatalf("dial failed: %v", err) } @@ -247,8 +146,9 @@ func (s *Suite) TestSimultaneousRequests66(t *utesting.T) { if err := conn.peer(s.chain, nil); err != nil { t.Fatalf("peering failed: %v", err) } + // create two requests - req1 := ð.GetBlockHeadersPacket66{ + req1 := &GetBlockHeaders{ RequestId: uint64(111), GetBlockHeadersPacket: ð.GetBlockHeadersPacket{ Origin: eth.HashOrNumber{ @@ -259,7 +159,7 @@ func (s *Suite) TestSimultaneousRequests66(t *utesting.T) { Reverse: false, }, } - req2 := ð.GetBlockHeadersPacket66{ + req2 := &GetBlockHeaders{ RequestId: uint64(222), GetBlockHeadersPacket: ð.GetBlockHeadersPacket{ Origin: eth.HashOrNumber{ @@ -270,46 +170,49 @@ func (s *Suite) TestSimultaneousRequests66(t *utesting.T) { Reverse: false, }, } + // write the first request - if err := conn.Write66(req1, GetBlockHeaders{}.Code()); err != nil { + if err := conn.Write(req1); err != nil { t.Fatalf("failed to write to connection: %v", err) } // write the second request - if err := conn.Write66(req2, GetBlockHeaders{}.Code()); err != nil { + if err := conn.Write(req2); err != nil { t.Fatalf("failed to write to connection: %v", err) } + // wait for responses msg := conn.waitForResponse(s.chain, timeout, req1.RequestId) - headers1, ok := msg.(BlockHeaders) + headers1, ok := msg.(*BlockHeaders) if !ok { t.Fatalf("unexpected %s", pretty.Sdump(msg)) } msg = conn.waitForResponse(s.chain, timeout, req2.RequestId) - headers2, ok := msg.(BlockHeaders) + headers2, ok := msg.(*BlockHeaders) if !ok { t.Fatalf("unexpected %s", pretty.Sdump(msg)) } + // check received headers for accuracy - expected1, err := s.chain.GetHeaders(GetBlockHeaders(*req1.GetBlockHeadersPacket)) + expected1, err := s.chain.GetHeaders(req1) if err != nil { t.Fatalf("failed to get expected headers for request 1: %v", err) } - expected2, err := s.chain.GetHeaders(GetBlockHeaders(*req2.GetBlockHeadersPacket)) + expected2, err := s.chain.GetHeaders(req2) if err != nil { t.Fatalf("failed to get expected headers for request 2: %v", err) } - if !headersMatch(expected1, headers1) { + if !headersMatch(expected1, headers1.BlockHeadersPacket) { t.Fatalf("header mismatch: \nexpected %v \ngot %v", expected1, headers1) } - if !headersMatch(expected2, headers2) { + if !headersMatch(expected2, headers2.BlockHeadersPacket) { t.Fatalf("header mismatch: \nexpected %v \ngot %v", expected2, headers2) } } -// TestSameRequestID66 sends two requests with the same request ID to a +// TestSameRequestID sends two requests with the same request ID to a // single node. -func (s *Suite) TestSameRequestID66(t *utesting.T) { - conn, err := s.dial66() +func (s *Suite) TestSameRequestID(t *utesting.T) { + conn, err := s.dial() if err != nil { t.Fatalf("dial failed: %v", err) } @@ -319,7 +222,7 @@ func (s *Suite) TestSameRequestID66(t *utesting.T) { } // create requests reqID := uint64(1234) - request1 := ð.GetBlockHeadersPacket66{ + request1 := &GetBlockHeaders{ RequestId: reqID, GetBlockHeadersPacket: ð.GetBlockHeadersPacket{ Origin: eth.HashOrNumber{ @@ -328,7 +231,7 @@ func (s *Suite) TestSameRequestID66(t *utesting.T) { Amount: 2, }, } - request2 := ð.GetBlockHeadersPacket66{ + request2 := &GetBlockHeaders{ RequestId: reqID, GetBlockHeadersPacket: ð.GetBlockHeadersPacket{ Origin: eth.HashOrNumber{ @@ -337,45 +240,48 @@ func (s *Suite) TestSameRequestID66(t *utesting.T) { Amount: 2, }, } + // write the requests - if err = conn.Write66(request1, GetBlockHeaders{}.Code()); err != nil { + if err = conn.Write(request1); err != nil { t.Fatalf("failed to write to connection: %v", err) } - if err = conn.Write66(request2, GetBlockHeaders{}.Code()); err != nil { + if err = conn.Write(request2); err != nil { t.Fatalf("failed to write to connection: %v", err) } + // wait for responses msg := conn.waitForResponse(s.chain, timeout, reqID) - headers1, ok := msg.(BlockHeaders) + headers1, ok := msg.(*BlockHeaders) if !ok { t.Fatalf("unexpected %s", pretty.Sdump(msg)) } msg = conn.waitForResponse(s.chain, timeout, reqID) - headers2, ok := msg.(BlockHeaders) + headers2, ok := msg.(*BlockHeaders) if !ok { t.Fatalf("unexpected %s", pretty.Sdump(msg)) } + // check if headers match - expected1, err := s.chain.GetHeaders(GetBlockHeaders(*request1.GetBlockHeadersPacket)) + expected1, err := s.chain.GetHeaders(request1) if err != nil { t.Fatalf("failed to get expected block headers: %v", err) } - expected2, err := s.chain.GetHeaders(GetBlockHeaders(*request2.GetBlockHeadersPacket)) + expected2, err := s.chain.GetHeaders(request2) if err != nil { t.Fatalf("failed to get expected block headers: %v", err) } - if !headersMatch(expected1, headers1) { + if !headersMatch(expected1, headers1.BlockHeadersPacket) { t.Fatalf("header mismatch: \nexpected %v \ngot %v", expected1, headers1) } - if !headersMatch(expected2, headers2) { + if !headersMatch(expected2, headers2.BlockHeadersPacket) { t.Fatalf("header mismatch: \nexpected %v \ngot %v", expected2, headers2) } } -// TestZeroRequestID_66 checks that a message with a request ID of zero is still handled +// TestZeroRequestID checks that a message with a request ID of zero is still handled // by the node. -func (s *Suite) TestZeroRequestID66(t *utesting.T) { - conn, err := s.dial66() +func (s *Suite) TestZeroRequestID(t *utesting.T) { + conn, err := s.dial() if err != nil { t.Fatalf("dial failed: %v", err) } @@ -384,16 +290,16 @@ func (s *Suite) TestZeroRequestID66(t *utesting.T) { t.Fatalf("peering failed: %v", err) } req := &GetBlockHeaders{ - Origin: eth.HashOrNumber{ - Number: 0, + GetBlockHeadersPacket: ð.GetBlockHeadersPacket{ + Origin: eth.HashOrNumber{Number: 0}, + Amount: 2, }, - Amount: 2, } - headers, err := conn.headersRequest(req, s.chain, eth66, 0) + headers, err := conn.headersRequest(req, s.chain, 0) if err != nil { t.Fatalf("failed to get block headers: %v", err) } - expected, err := s.chain.GetHeaders(*req) + expected, err := s.chain.GetHeaders(req) if err != nil { t.Fatalf("failed to get expected block headers: %v", err) } @@ -402,9 +308,9 @@ func (s *Suite) TestZeroRequestID66(t *utesting.T) { } } -// TestGetBlockBodies65 tests whether the given node can respond to +// TestGetBlockBodies tests whether the given node can respond to // a `GetBlockBodies` request and that the response is accurate. -func (s *Suite) TestGetBlockBodies65(t *utesting.T) { +func (s *Suite) TestGetBlockBodies(t *utesting.T) { conn, err := s.dial() if err != nil { t.Fatalf("dial failed: %v", err) @@ -415,126 +321,39 @@ func (s *Suite) TestGetBlockBodies65(t *utesting.T) { } // create block bodies request req := &GetBlockBodies{ - s.chain.blocks[54].Hash(), - s.chain.blocks[75].Hash(), - } - if err := conn.Write(req); err != nil { - t.Fatalf("could not write to connection: %v", err) - } - // wait for response - switch msg := conn.readAndServe(s.chain, timeout).(type) { - case *BlockBodies: - t.Logf("received %d block bodies", len(*msg)) - if len(*msg) != len(*req) { - t.Fatalf("wrong bodies in response: expected %d bodies, "+ - "got %d", len(*req), len(*msg)) - } - default: - t.Fatalf("unexpected: %s", pretty.Sdump(msg)) - } -} - -// TestGetBlockBodies66 tests whether the given node can respond to -// a `GetBlockBodies` request and that the response is accurate over -// the eth66 protocol. -func (s *Suite) TestGetBlockBodies66(t *utesting.T) { - conn, err := s.dial66() - if err != nil { - t.Fatalf("dial failed: %v", err) - } - defer conn.Close() - if err := conn.peer(s.chain, nil); err != nil { - t.Fatalf("peering failed: %v", err) - } - // create block bodies request - req := ð.GetBlockBodiesPacket66{ RequestId: uint64(55), GetBlockBodiesPacket: eth.GetBlockBodiesPacket{ s.chain.blocks[54].Hash(), s.chain.blocks[75].Hash(), }, } - if err := conn.Write66(req, GetBlockBodies{}.Code()); err != nil { + if err := conn.Write(req); err != nil { t.Fatalf("could not write to connection: %v", err) } // wait for block bodies response msg := conn.waitForResponse(s.chain, timeout, req.RequestId) - blockBodies, ok := msg.(BlockBodies) + resp, ok := msg.(*BlockBodies) if !ok { t.Fatalf("unexpected: %s", pretty.Sdump(msg)) } - t.Logf("received %d block bodies", len(blockBodies)) - if len(blockBodies) != len(req.GetBlockBodiesPacket) { + bodies := resp.BlockBodiesPacket + t.Logf("received %d block bodies", len(bodies)) + if len(bodies) != len(req.GetBlockBodiesPacket) { t.Fatalf("wrong bodies in response: expected %d bodies, "+ - "got %d", len(req.GetBlockBodiesPacket), len(blockBodies)) - } -} - -// TestBroadcast65 tests whether a block announcement is correctly -// propagated to the given node's peer(s). -func (s *Suite) TestBroadcast65(t *utesting.T) { - if err := s.sendNextBlock(eth65); err != nil { - t.Fatalf("block broadcast failed: %v", err) + "got %d", len(req.GetBlockBodiesPacket), len(bodies)) } } -// TestBroadcast66 tests whether a block announcement is correctly -// propagated to the given node's peer(s) on the eth66 protocol. -func (s *Suite) TestBroadcast66(t *utesting.T) { - if err := s.sendNextBlock(eth66); err != nil { +// TestBroadcast tests whether a block announcement is correctly +// propagated to the node's peers. +func (s *Suite) TestBroadcast(t *utesting.T) { + if err := s.sendNextBlock(); err != nil { t.Fatalf("block broadcast failed: %v", err) } } -// TestLargeAnnounce65 tests the announcement mechanism with a large block. -func (s *Suite) TestLargeAnnounce65(t *utesting.T) { - nextBlock := len(s.chain.blocks) - blocks := []*NewBlock{ - { - Block: largeBlock(), - TD: s.fullChain.TotalDifficultyAt(nextBlock), - }, - { - Block: s.fullChain.blocks[nextBlock], - TD: largeNumber(2), - }, - { - Block: largeBlock(), - TD: largeNumber(2), - }, - } - - for i, blockAnnouncement := range blocks { - t.Logf("Testing malicious announcement: %v\n", i) - conn, err := s.dial() - if err != nil { - t.Fatalf("dial failed: %v", err) - } - if err = conn.peer(s.chain, nil); err != nil { - t.Fatalf("peering failed: %v", err) - } - if err = conn.Write(blockAnnouncement); err != nil { - t.Fatalf("could not write to connection: %v", err) - } - // Invalid announcement, check that peer disconnected - switch msg := conn.readAndServe(s.chain, time.Second*8).(type) { - case *Disconnect: - case *Error: - break - default: - t.Fatalf("unexpected: %s wanted disconnect", pretty.Sdump(msg)) - } - conn.Close() - } - // Test the last block as a valid block - if err := s.sendNextBlock(eth65); err != nil { - t.Fatalf("failed to broadcast next block: %v", err) - } -} - -// TestLargeAnnounce66 tests the announcement mechanism with a large -// block over the eth66 protocol. -func (s *Suite) TestLargeAnnounce66(t *utesting.T) { +// TestLargeAnnounce tests the announcement mechanism with a large block. +func (s *Suite) TestLargeAnnounce(t *utesting.T) { nextBlock := len(s.chain.blocks) blocks := []*NewBlock{ { @@ -553,7 +372,7 @@ func (s *Suite) TestLargeAnnounce66(t *utesting.T) { for i, blockAnnouncement := range blocks[0:3] { t.Logf("Testing malicious announcement: %v\n", i) - conn, err := s.dial66() + conn, err := s.dial() if err != nil { t.Fatalf("dial failed: %v", err) } @@ -564,7 +383,7 @@ func (s *Suite) TestLargeAnnounce66(t *utesting.T) { t.Fatalf("could not write to connection: %v", err) } // Invalid announcement, check that peer disconnected - switch msg := conn.readAndServe(s.chain, time.Second*8).(type) { + switch msg := conn.readAndServe(s.chain, 8*time.Second).(type) { case *Disconnect: case *Error: break @@ -574,58 +393,35 @@ func (s *Suite) TestLargeAnnounce66(t *utesting.T) { conn.Close() } // Test the last block as a valid block - if err := s.sendNextBlock(eth66); err != nil { + if err := s.sendNextBlock(); err != nil { t.Fatalf("failed to broadcast next block: %v", err) } } -// TestOldAnnounce65 tests the announcement mechanism with an old block. -func (s *Suite) TestOldAnnounce65(t *utesting.T) { - if err := s.oldAnnounce(eth65); err != nil { - t.Fatal(err) - } -} - -// TestOldAnnounce66 tests the announcement mechanism with an old block, -// over the eth66 protocol. -func (s *Suite) TestOldAnnounce66(t *utesting.T) { - if err := s.oldAnnounce(eth66); err != nil { +// TestOldAnnounce tests the announcement mechanism with an old block. +func (s *Suite) TestOldAnnounce(t *utesting.T) { + if err := s.oldAnnounce(); err != nil { t.Fatal(err) } } -// TestBlockHashAnnounce65 sends a new block hash announcement and expects -// the node to perform a `GetBlockHeaders` request. -func (s *Suite) TestBlockHashAnnounce65(t *utesting.T) { - if err := s.hashAnnounce(eth65); err != nil { - t.Fatalf("block hash announcement failed: %v", err) - } -} - -// TestBlockHashAnnounce66 sends a new block hash announcement and expects +// TestBlockHashAnnounce sends a new block hash announcement and expects // the node to perform a `GetBlockHeaders` request. -func (s *Suite) TestBlockHashAnnounce66(t *utesting.T) { - if err := s.hashAnnounce(eth66); err != nil { +func (s *Suite) TestBlockHashAnnounce(t *utesting.T) { + if err := s.hashAnnounce(); err != nil { t.Fatalf("block hash announcement failed: %v", err) } } -// TestMaliciousHandshake65 tries to send malicious data during the handshake. -func (s *Suite) TestMaliciousHandshake65(t *utesting.T) { - if err := s.maliciousHandshakes(t, eth65); err != nil { - t.Fatal(err) - } -} - -// TestMaliciousHandshake66 tries to send malicious data during the handshake. -func (s *Suite) TestMaliciousHandshake66(t *utesting.T) { - if err := s.maliciousHandshakes(t, eth66); err != nil { +// TestMaliciousHandshake tries to send malicious data during the handshake. +func (s *Suite) TestMaliciousHandshake(t *utesting.T) { + if err := s.maliciousHandshakes(t); err != nil { t.Fatal(err) } } -// TestMaliciousStatus65 sends a status package with a large total difficulty. -func (s *Suite) TestMaliciousStatus65(t *utesting.T) { +// TestMaliciousStatus sends a status package with a large total difficulty. +func (s *Suite) TestMaliciousStatus(t *utesting.T) { conn, err := s.dial() if err != nil { t.Fatalf("dial failed: %v", err) @@ -637,58 +433,28 @@ func (s *Suite) TestMaliciousStatus65(t *utesting.T) { } } -// TestMaliciousStatus66 sends a status package with a large total -// difficulty over the eth66 protocol. -func (s *Suite) TestMaliciousStatus66(t *utesting.T) { - conn, err := s.dial66() - if err != nil { - t.Fatalf("dial failed: %v", err) - } - defer conn.Close() - - if err := s.maliciousStatus(conn); err != nil { - t.Fatal(err) - } -} - -// TestTransaction65 sends a valid transaction to the node and +// TestTransaction sends a valid transaction to the node and // checks if the transaction gets propagated. -func (s *Suite) TestTransaction65(t *utesting.T) { - if err := s.sendSuccessfulTxs(t, eth65); err != nil { +func (s *Suite) TestTransaction(t *utesting.T) { + if err := s.sendSuccessfulTxs(t); err != nil { t.Fatal(err) } } -// TestTransaction66 sends a valid transaction to the node and -// checks if the transaction gets propagated. -func (s *Suite) TestTransaction66(t *utesting.T) { - if err := s.sendSuccessfulTxs(t, eth66); err != nil { - t.Fatal(err) - } -} - -// TestMaliciousTx65 sends several invalid transactions and tests whether +// TestMaliciousTx sends several invalid transactions and tests whether // the node will propagate them. -func (s *Suite) TestMaliciousTx65(t *utesting.T) { - if err := s.sendMaliciousTxs(t, eth65); err != nil { +func (s *Suite) TestMaliciousTx(t *utesting.T) { + if err := s.sendMaliciousTxs(t); err != nil { t.Fatal(err) } } -// TestMaliciousTx66 sends several invalid transactions and tests whether -// the node will propagate them. -func (s *Suite) TestMaliciousTx66(t *utesting.T) { - if err := s.sendMaliciousTxs(t, eth66); err != nil { - t.Fatal(err) - } -} - -// TestLargeTxRequest66 tests whether a node can fulfill a large GetPooledTransactions +// TestLargeTxRequest tests whether a node can fulfill a large GetPooledTransactions // request. -func (s *Suite) TestLargeTxRequest66(t *utesting.T) { +func (s *Suite) TestLargeTxRequest(t *utesting.T) { // send the next block to ensure the node is no longer syncing and // is able to accept txs - if err := s.sendNextBlock(eth66); err != nil { + if err := s.sendNextBlock(); err != nil { t.Fatalf("failed to send next block: %v", err) } // send 2000 transactions to the node @@ -701,7 +467,7 @@ func (s *Suite) TestLargeTxRequest66(t *utesting.T) { } // set up connection to receive to ensure node is peered with the receiving connection // before tx request is sent - conn, err := s.dial66() + conn, err := s.dial() if err != nil { t.Fatalf("dial failed: %v", err) } @@ -714,17 +480,17 @@ func (s *Suite) TestLargeTxRequest66(t *utesting.T) { for _, hash := range hashMap { hashes = append(hashes, hash) } - getTxReq := ð.GetPooledTransactionsPacket66{ + getTxReq := &GetPooledTransactions{ RequestId: 1234, GetPooledTransactionsPacket: hashes, } - if err = conn.Write66(getTxReq, GetPooledTransactions{}.Code()); err != nil { + if err = conn.Write(getTxReq); err != nil { t.Fatalf("could not write to conn: %v", err) } // check that all received transactions match those that were sent to node switch msg := conn.waitForResponse(s.chain, timeout, getTxReq.RequestId).(type) { - case PooledTransactions: - for _, gotTx := range msg { + case *PooledTransactions: + for _, gotTx := range msg.PooledTransactionsPacket { if _, exists := hashMap[gotTx.Hash()]; !exists { t.Fatalf("unexpected tx received: %v", gotTx.Hash()) } @@ -734,12 +500,12 @@ func (s *Suite) TestLargeTxRequest66(t *utesting.T) { } } -// TestNewPooledTxs_66 tests whether a node will do a GetPooledTransactions +// TestNewPooledTxs tests whether a node will do a GetPooledTransactions // request upon receiving a NewPooledTransactionHashes announcement. -func (s *Suite) TestNewPooledTxs66(t *utesting.T) { +func (s *Suite) TestNewPooledTxs(t *utesting.T) { // send the next block to ensure the node is no longer syncing and // is able to accept txs - if err := s.sendNextBlock(eth66); err != nil { + if err := s.sendNextBlock(); err != nil { t.Fatalf("failed to send next block: %v", err) } @@ -757,7 +523,7 @@ func (s *Suite) TestNewPooledTxs66(t *utesting.T) { announce := NewPooledTransactionHashes(hashes) // send announcement - conn, err := s.dial66() + conn, err := s.dial() if err != nil { t.Fatalf("dial failed: %v", err) } @@ -771,11 +537,11 @@ func (s *Suite) TestNewPooledTxs66(t *utesting.T) { // wait for GetPooledTxs request for { - _, msg := conn.readAndServe66(s.chain, timeout) + msg := conn.readAndServe(s.chain, timeout) switch msg := msg.(type) { - case GetPooledTransactions: - if len(msg) != len(hashes) { - t.Fatalf("unexpected number of txs requested: wanted %d, got %d", len(hashes), len(msg)) + case *GetPooledTransactions: + if len(msg.GetPooledTransactionsPacket) != len(hashes) { + t.Fatalf("unexpected number of txs requested: wanted %d, got %d", len(hashes), len(msg.GetPooledTransactionsPacket)) } return // ignore propagated txs from previous tests diff --git a/cmd/devp2p/internal/ethtest/suite_test.go b/cmd/devp2p/internal/ethtest/suite_test.go index 924c80d01c8cb..8a2b132fa3b15 100644 --- a/cmd/devp2p/internal/ethtest/suite_test.go +++ b/cmd/devp2p/internal/ethtest/suite_test.go @@ -45,7 +45,7 @@ func TestEthSuite(t *testing.T) { if err != nil { t.Fatalf("could not create new test suite: %v", err) } - for _, test := range suite.Eth66Tests() { + for _, test := range suite.EthTests() { t.Run(test.Name, func(t *testing.T) { result := utesting.RunTAP([]utesting.Test{{Name: test.Name, Fn: test.Fn}}, os.Stdout) if result[0].Failed { diff --git a/cmd/devp2p/internal/ethtest/transaction.go b/cmd/devp2p/internal/ethtest/transaction.go index 5d722f417a223..c4748bf8f7d88 100644 --- a/cmd/devp2p/internal/ethtest/transaction.go +++ b/cmd/devp2p/internal/ethtest/transaction.go @@ -32,7 +32,7 @@ import ( //var faucetAddr = common.HexToAddress("0x71562b71999873DB5b286dF957af199Ec94617F7") var faucetKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") -func (s *Suite) sendSuccessfulTxs(t *utesting.T, isEth66 bool) error { +func (s *Suite) sendSuccessfulTxs(t *utesting.T) error { tests := []*types.Transaction{ getNextTxFromChain(s), unknownTx(s), @@ -48,15 +48,15 @@ func (s *Suite) sendSuccessfulTxs(t *utesting.T, isEth66 bool) error { prevTx = tests[i-1] } // write tx to connection - if err := sendSuccessfulTx(s, tx, prevTx, isEth66); err != nil { + if err := sendSuccessfulTx(s, tx, prevTx); err != nil { return fmt.Errorf("send successful tx test failed: %v", err) } } return nil } -func sendSuccessfulTx(s *Suite, tx *types.Transaction, prevTx *types.Transaction, isEth66 bool) error { - sendConn, recvConn, err := s.createSendAndRecvConns(isEth66) +func sendSuccessfulTx(s *Suite, tx *types.Transaction, prevTx *types.Transaction) error { + sendConn, recvConn, err := s.createSendAndRecvConns() if err != nil { return err } @@ -73,8 +73,10 @@ func sendSuccessfulTx(s *Suite, tx *types.Transaction, prevTx *types.Transaction if err = recvConn.peer(s.chain, nil); err != nil { return fmt.Errorf("peering failed: %v", err) } + // update last nonce seen nonce = tx.Nonce() + // Wait for the transaction announcement for { switch msg := recvConn.readAndServe(s.chain, timeout).(type) { @@ -114,7 +116,7 @@ func sendSuccessfulTx(s *Suite, tx *types.Transaction, prevTx *types.Transaction } } -func (s *Suite) sendMaliciousTxs(t *utesting.T, isEth66 bool) error { +func (s *Suite) sendMaliciousTxs(t *utesting.T) error { badTxs := []*types.Transaction{ getOldTxFromChain(s), invalidNonceTx(s), @@ -122,16 +124,9 @@ func (s *Suite) sendMaliciousTxs(t *utesting.T, isEth66 bool) error { hugeGasPrice(s), hugeData(s), } + // setup receiving connection before sending malicious txs - var ( - recvConn *Conn - err error - ) - if isEth66 { - recvConn, err = s.dial66() - } else { - recvConn, err = s.dial() - } + recvConn, err := s.dial() if err != nil { return fmt.Errorf("dial failed: %v", err) } @@ -139,9 +134,10 @@ func (s *Suite) sendMaliciousTxs(t *utesting.T, isEth66 bool) error { if err = recvConn.peer(s.chain, nil); err != nil { return fmt.Errorf("peering failed: %v", err) } + for i, tx := range badTxs { t.Logf("Testing malicious tx propagation: %v\n", i) - if err = sendMaliciousTx(s, tx, isEth66); err != nil { + if err = sendMaliciousTx(s, tx); err != nil { return fmt.Errorf("malicious tx test failed:\ntx: %v\nerror: %v", tx, err) } } @@ -149,17 +145,8 @@ func (s *Suite) sendMaliciousTxs(t *utesting.T, isEth66 bool) error { return checkMaliciousTxPropagation(s, badTxs, recvConn) } -func sendMaliciousTx(s *Suite, tx *types.Transaction, isEth66 bool) error { - // setup connection - var ( - conn *Conn - err error - ) - if isEth66 { - conn, err = s.dial66() - } else { - conn, err = s.dial() - } +func sendMaliciousTx(s *Suite, tx *types.Transaction) error { + conn, err := s.dial() if err != nil { return fmt.Errorf("dial failed: %v", err) } @@ -167,6 +154,7 @@ func sendMaliciousTx(s *Suite, tx *types.Transaction, isEth66 bool) error { if err = conn.peer(s.chain, nil); err != nil { return fmt.Errorf("peering failed: %v", err) } + // write malicious tx if err = conn.Write(&Transactions{tx}); err != nil { return fmt.Errorf("failed to write to connection: %v", err) @@ -182,7 +170,7 @@ func sendMultipleSuccessfulTxs(t *utesting.T, s *Suite, txs []*types.Transaction txMsg := Transactions(txs) t.Logf("sending %d txs\n", len(txs)) - sendConn, recvConn, err := s.createSendAndRecvConns(true) + sendConn, recvConn, err := s.createSendAndRecvConns() if err != nil { return err } @@ -194,15 +182,19 @@ func sendMultipleSuccessfulTxs(t *utesting.T, s *Suite, txs []*types.Transaction if err = recvConn.peer(s.chain, nil); err != nil { return fmt.Errorf("peering failed: %v", err) } + // Send the transactions if err = sendConn.Write(&txMsg); err != nil { return fmt.Errorf("failed to write message to connection: %v", err) } + // update nonce nonce = txs[len(txs)-1].Nonce() - // Wait for the transaction announcement(s) and make sure all sent txs are being propagated + + // Wait for the transaction announcement(s) and make sure all sent txs are being propagated. + // all txs should be announced within 3 announcements. recvHashes := make([]common.Hash, 0) - // all txs should be announced within 3 announcements + for i := 0; i < 3; i++ { switch msg := recvConn.readAndServe(s.chain, timeout).(type) { case *Transactions: diff --git a/cmd/devp2p/internal/ethtest/types.go b/cmd/devp2p/internal/ethtest/types.go index e92b543940679..2c5cb94c699f1 100644 --- a/cmd/devp2p/internal/ethtest/types.go +++ b/cmd/devp2p/internal/ethtest/types.go @@ -29,6 +29,7 @@ import ( type Message interface { Code() int + ReqID() uint64 } type Error struct { @@ -37,9 +38,11 @@ type Error struct { func (e *Error) Unwrap() error { return e.err } func (e *Error) Error() string { return e.err.Error() } -func (e *Error) Code() int { return -1 } func (e *Error) String() string { return e.Error() } +func (e *Error) Code() int { return -1 } +func (e *Error) ReqID() uint64 { return 0 } + func errorf(format string, args ...interface{}) *Error { return &Error{fmt.Errorf(format, args...)} } @@ -56,73 +59,88 @@ type Hello struct { Rest []rlp.RawValue `rlp:"tail"` } -func (h Hello) Code() int { return 0x00 } +func (msg Hello) Code() int { return 0x00 } +func (msg Hello) ReqID() uint64 { return 0 } // Disconnect is the RLP structure for a disconnect message. type Disconnect struct { Reason p2p.DiscReason } -func (d Disconnect) Code() int { return 0x01 } +func (msg Disconnect) Code() int { return 0x01 } +func (msg Disconnect) ReqID() uint64 { return 0 } type Ping struct{} -func (p Ping) Code() int { return 0x02 } +func (msg Ping) Code() int { return 0x02 } +func (msg Ping) ReqID() uint64 { return 0 } type Pong struct{} -func (p Pong) Code() int { return 0x03 } +func (msg Pong) Code() int { return 0x03 } +func (msg Pong) ReqID() uint64 { return 0 } // Status is the network packet for the status message for eth/64 and later. type Status eth.StatusPacket -func (s Status) Code() int { return 16 } +func (msg Status) Code() int { return 16 } +func (msg Status) ReqID() uint64 { return 0 } // NewBlockHashes is the network packet for the block announcements. type NewBlockHashes eth.NewBlockHashesPacket -func (nbh NewBlockHashes) Code() int { return 17 } +func (msg NewBlockHashes) Code() int { return 17 } +func (msg NewBlockHashes) ReqID() uint64 { return 0 } type Transactions eth.TransactionsPacket -func (t Transactions) Code() int { return 18 } +func (msg Transactions) Code() int { return 18 } +func (msg Transactions) ReqID() uint64 { return 18 } // GetBlockHeaders represents a block header query. -type GetBlockHeaders eth.GetBlockHeadersPacket +type GetBlockHeaders eth.GetBlockHeadersPacket66 -func (g GetBlockHeaders) Code() int { return 19 } +func (msg GetBlockHeaders) Code() int { return 19 } +func (msg GetBlockHeaders) ReqID() uint64 { return msg.RequestId } -type BlockHeaders eth.BlockHeadersPacket +type BlockHeaders eth.BlockHeadersPacket66 -func (bh BlockHeaders) Code() int { return 20 } +func (msg BlockHeaders) Code() int { return 20 } +func (msg BlockHeaders) ReqID() uint64 { return msg.RequestId } // GetBlockBodies represents a GetBlockBodies request -type GetBlockBodies eth.GetBlockBodiesPacket +type GetBlockBodies eth.GetBlockBodiesPacket66 -func (gbb GetBlockBodies) Code() int { return 21 } +func (msg GetBlockBodies) Code() int { return 21 } +func (msg GetBlockBodies) ReqID() uint64 { return msg.RequestId } // BlockBodies is the network packet for block content distribution. -type BlockBodies eth.BlockBodiesPacket +type BlockBodies eth.BlockBodiesPacket66 -func (bb BlockBodies) Code() int { return 22 } +func (msg BlockBodies) Code() int { return 22 } +func (msg BlockBodies) ReqID() uint64 { return msg.RequestId } // NewBlock is the network packet for the block propagation message. type NewBlock eth.NewBlockPacket -func (nb NewBlock) Code() int { return 23 } +func (msg NewBlock) Code() int { return 23 } +func (msg NewBlock) ReqID() uint64 { return 0 } // NewPooledTransactionHashes is the network packet for the tx hash propagation message. type NewPooledTransactionHashes eth.NewPooledTransactionHashesPacket -func (nb NewPooledTransactionHashes) Code() int { return 24 } +func (msg NewPooledTransactionHashes) Code() int { return 24 } +func (msg NewPooledTransactionHashes) ReqID() uint64 { return 0 } -type GetPooledTransactions eth.GetPooledTransactionsPacket +type GetPooledTransactions eth.GetPooledTransactionsPacket66 -func (gpt GetPooledTransactions) Code() int { return 25 } +func (msg GetPooledTransactions) Code() int { return 25 } +func (msg GetPooledTransactions) ReqID() uint64 { return msg.RequestId } -type PooledTransactions eth.PooledTransactionsPacket +type PooledTransactions eth.PooledTransactionsPacket66 -func (pt PooledTransactions) Code() int { return 26 } +func (msg PooledTransactions) Code() int { return 26 } +func (msg PooledTransactions) ReqID() uint64 { return msg.RequestId } // Conn represents an individual connection with a peer type Conn struct { @@ -135,62 +153,13 @@ type Conn struct { caps []p2p.Cap } -// Read reads an eth packet from the connection. +// Read reads an eth66 packet from the connection. func (c *Conn) Read() Message { code, rawData, _, err := c.Conn.Read() if err != nil { return errorf("could not read from connection: %v", err) } - var msg Message - switch int(code) { - case (Hello{}).Code(): - msg = new(Hello) - case (Ping{}).Code(): - msg = new(Ping) - case (Pong{}).Code(): - msg = new(Pong) - case (Disconnect{}).Code(): - msg = new(Disconnect) - case (Status{}).Code(): - msg = new(Status) - case (GetBlockHeaders{}).Code(): - msg = new(GetBlockHeaders) - case (BlockHeaders{}).Code(): - msg = new(BlockHeaders) - case (GetBlockBodies{}).Code(): - msg = new(GetBlockBodies) - case (BlockBodies{}).Code(): - msg = new(BlockBodies) - case (NewBlock{}).Code(): - msg = new(NewBlock) - case (NewBlockHashes{}).Code(): - msg = new(NewBlockHashes) - case (Transactions{}).Code(): - msg = new(Transactions) - case (NewPooledTransactionHashes{}).Code(): - msg = new(NewPooledTransactionHashes) - case (GetPooledTransactions{}.Code()): - msg = new(GetPooledTransactions) - case (PooledTransactions{}.Code()): - msg = new(PooledTransactions) - default: - return errorf("invalid message code: %d", code) - } - // if message is devp2p, decode here - if err := rlp.DecodeBytes(rawData, msg); err != nil { - return errorf("could not rlp decode message: %v", err) - } - return msg -} - -// Read66 reads an eth66 packet from the connection. -func (c *Conn) Read66() (uint64, Message) { - code, rawData, _, err := c.Conn.Read() - if err != nil { - return 0, errorf("could not read from connection: %v", err) - } - var msg Message switch int(code) { case (Hello{}).Code(): @@ -206,27 +175,27 @@ func (c *Conn) Read66() (uint64, Message) { case (GetBlockHeaders{}).Code(): ethMsg := new(eth.GetBlockHeadersPacket66) if err := rlp.DecodeBytes(rawData, ethMsg); err != nil { - return 0, errorf("could not rlp decode message: %v", err) + return errorf("could not rlp decode message: %v", err) } - return ethMsg.RequestId, GetBlockHeaders(*ethMsg.GetBlockHeadersPacket) + return (*GetBlockHeaders)(ethMsg) case (BlockHeaders{}).Code(): ethMsg := new(eth.BlockHeadersPacket66) if err := rlp.DecodeBytes(rawData, ethMsg); err != nil { - return 0, errorf("could not rlp decode message: %v", err) + return errorf("could not rlp decode message: %v", err) } - return ethMsg.RequestId, BlockHeaders(ethMsg.BlockHeadersPacket) + return (*BlockHeaders)(ethMsg) case (GetBlockBodies{}).Code(): ethMsg := new(eth.GetBlockBodiesPacket66) if err := rlp.DecodeBytes(rawData, ethMsg); err != nil { - return 0, errorf("could not rlp decode message: %v", err) + return errorf("could not rlp decode message: %v", err) } - return ethMsg.RequestId, GetBlockBodies(ethMsg.GetBlockBodiesPacket) + return (*GetBlockBodies)(ethMsg) case (BlockBodies{}).Code(): ethMsg := new(eth.BlockBodiesPacket66) if err := rlp.DecodeBytes(rawData, ethMsg); err != nil { - return 0, errorf("could not rlp decode message: %v", err) + return errorf("could not rlp decode message: %v", err) } - return ethMsg.RequestId, BlockBodies(ethMsg.BlockBodiesPacket) + return (*BlockBodies)(ethMsg) case (NewBlock{}).Code(): msg = new(NewBlock) case (NewBlockHashes{}).Code(): @@ -238,26 +207,26 @@ func (c *Conn) Read66() (uint64, Message) { case (GetPooledTransactions{}.Code()): ethMsg := new(eth.GetPooledTransactionsPacket66) if err := rlp.DecodeBytes(rawData, ethMsg); err != nil { - return 0, errorf("could not rlp decode message: %v", err) + return errorf("could not rlp decode message: %v", err) } - return ethMsg.RequestId, GetPooledTransactions(ethMsg.GetPooledTransactionsPacket) + return (*GetPooledTransactions)(ethMsg) case (PooledTransactions{}.Code()): ethMsg := new(eth.PooledTransactionsPacket66) if err := rlp.DecodeBytes(rawData, ethMsg); err != nil { - return 0, errorf("could not rlp decode message: %v", err) + return errorf("could not rlp decode message: %v", err) } - return ethMsg.RequestId, PooledTransactions(ethMsg.PooledTransactionsPacket) + return (*PooledTransactions)(ethMsg) default: msg = errorf("invalid message code: %d", code) } if msg != nil { if err := rlp.DecodeBytes(rawData, msg); err != nil { - return 0, errorf("could not rlp decode message: %v", err) + return errorf("could not rlp decode message: %v", err) } - return 0, msg + return msg } - return 0, errorf("invalid message: %s", string(rawData)) + return errorf("invalid message: %s", string(rawData)) } // Write writes a eth packet to the connection. @@ -270,16 +239,6 @@ func (c *Conn) Write(msg Message) error { return err } -// Write66 writes an eth66 packet to the connection. -func (c *Conn) Write66(req eth.Packet, code int) error { - payload, err := rlp.EncodeToBytes(req) - if err != nil { - return err - } - _, err = c.Conn.Write(uint64(code), payload) - return err -} - // ReadSnap reads a snap/1 response with the given id from the connection. func (c *Conn) ReadSnap(id uint64) (Message, error) { respId := id + 1 @@ -315,7 +274,6 @@ func (c *Conn) ReadSnap(id uint64) (Message, error) { return nil, fmt.Errorf("could not rlp decode message: %v", err) } return snpMsg.(Message), nil - } return nil, fmt.Errorf("request timed out") } diff --git a/cmd/devp2p/internal/v5test/framework.go b/cmd/devp2p/internal/v5test/framework.go index 9eac37520f7b7..6ccbbd075bf0f 100644 --- a/cmd/devp2p/internal/v5test/framework.go +++ b/cmd/devp2p/internal/v5test/framework.go @@ -60,11 +60,9 @@ type conn struct { remoteAddr *net.UDPAddr listeners []net.PacketConn - log logger - codec *v5wire.Codec - lastRequest v5wire.Packet - lastChallenge *v5wire.Whoareyou - idCounter uint32 + log logger + codec *v5wire.Codec + idCounter uint32 } type logger interface { diff --git a/cmd/devp2p/keycmd.go b/cmd/devp2p/keycmd.go index 869b8c2a44f0f..e824abe653e2b 100644 --- a/cmd/devp2p/keycmd.go +++ b/cmd/devp2p/keycmd.go @@ -22,25 +22,25 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/p2p/enode" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) var ( - keyCommand = cli.Command{ + keyCommand = &cli.Command{ Name: "key", Usage: "Operations on node keys", - Subcommands: []cli.Command{ + Subcommands: []*cli.Command{ keyGenerateCommand, keyToNodeCommand, }, } - keyGenerateCommand = cli.Command{ + keyGenerateCommand = &cli.Command{ Name: "generate", Usage: "Generates node key files", ArgsUsage: "keyfile", Action: genkey, } - keyToNodeCommand = cli.Command{ + keyToNodeCommand = &cli.Command{ Name: "to-enode", Usage: "Creates an enode URL from a node key file", ArgsUsage: "keyfile", @@ -50,17 +50,17 @@ var ( ) var ( - hostFlag = cli.StringFlag{ + hostFlag = &cli.StringFlag{ Name: "ip", Usage: "IP address of the node", Value: "127.0.0.1", } - tcpPortFlag = cli.IntFlag{ + tcpPortFlag = &cli.IntFlag{ Name: "tcp", Usage: "TCP port of the node", Value: 30303, } - udpPortFlag = cli.IntFlag{ + udpPortFlag = &cli.IntFlag{ Name: "udp", Usage: "UDP port of the node", Value: 30303, diff --git a/cmd/devp2p/main.go b/cmd/devp2p/main.go index 4a4e905a424ec..51b9fdb76119e 100644 --- a/cmd/devp2p/main.go +++ b/cmd/devp2p/main.go @@ -20,12 +20,12 @@ import ( "fmt" "os" "path/filepath" - "sort" "github.com/ethereum/go-ethereum/internal/debug" + "github.com/ethereum/go-ethereum/internal/flags" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/params" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) var ( @@ -45,6 +45,7 @@ func init() { // Set up the CLI app. app.Flags = append(app.Flags, debug.Flags...) app.Before = func(ctx *cli.Context) error { + flags.MigrateGlobalFlags(ctx) return debug.Setup(ctx) } app.After = func(ctx *cli.Context) error { @@ -56,7 +57,7 @@ func init() { os.Exit(1) } // Add subcommands. - app.Commands = []cli.Command{ + app.Commands = []*cli.Command{ enrdumpCommand, keyCommand, discv4Command, @@ -73,10 +74,17 @@ func main() { // commandHasFlag returns true if the current command supports the given flag. func commandHasFlag(ctx *cli.Context, flag cli.Flag) bool { - flags := ctx.FlagNames() - sort.Strings(flags) - i := sort.SearchStrings(flags, flag.GetName()) - return i != len(flags) && flags[i] == flag.GetName() + names := flag.Names() + set := make(map[string]struct{}, len(names)) + for _, name := range names { + set[name] = struct{}{} + } + for _, fn := range ctx.FlagNames() { + if _, ok := set[fn]; ok { + return true + } + } + return false } // getNodeArg handles the common case of a single node descriptor argument. @@ -84,7 +92,7 @@ func getNodeArg(ctx *cli.Context) *enode.Node { if ctx.NArg() < 1 { exit("missing node as command-line argument") } - n, err := parseNode(ctx.Args()[0]) + n, err := parseNode(ctx.Args().First()) if err != nil { exit(err) } diff --git a/cmd/devp2p/nodesetcmd.go b/cmd/devp2p/nodesetcmd.go index d65d6314c8e13..f6b629278ecdc 100644 --- a/cmd/devp2p/nodesetcmd.go +++ b/cmd/devp2p/nodesetcmd.go @@ -29,25 +29,25 @@ import ( "github.com/ethereum/go-ethereum/p2p/enr" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) var ( - nodesetCommand = cli.Command{ + nodesetCommand = &cli.Command{ Name: "nodeset", Usage: "Node set tools", - Subcommands: []cli.Command{ + Subcommands: []*cli.Command{ nodesetInfoCommand, nodesetFilterCommand, }, } - nodesetInfoCommand = cli.Command{ + nodesetInfoCommand = &cli.Command{ Name: "info", Usage: "Shows statistics about a node set", Action: nodesetInfo, ArgsUsage: "", } - nodesetFilterCommand = cli.Command{ + nodesetFilterCommand = &cli.Command{ Name: "filter", Usage: "Filters a node set", Action: nodesetFilter, diff --git a/cmd/devp2p/rlpxcmd.go b/cmd/devp2p/rlpxcmd.go index 6557a239da771..42b38120c475a 100644 --- a/cmd/devp2p/rlpxcmd.go +++ b/cmd/devp2p/rlpxcmd.go @@ -22,29 +22,28 @@ import ( "github.com/ethereum/go-ethereum/cmd/devp2p/internal/ethtest" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/internal/utesting" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/rlpx" "github.com/ethereum/go-ethereum/rlp" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) var ( - rlpxCommand = cli.Command{ + rlpxCommand = &cli.Command{ Name: "rlpx", Usage: "RLPx Commands", - Subcommands: []cli.Command{ + Subcommands: []*cli.Command{ rlpxPingCommand, rlpxEthTestCommand, rlpxSnapTestCommand, }, } - rlpxPingCommand = cli.Command{ + rlpxPingCommand = &cli.Command{ Name: "ping", Usage: "ping ", Action: rlpxPing, } - rlpxEthTestCommand = cli.Command{ + rlpxEthTestCommand = &cli.Command{ Name: "eth-test", Usage: "Runs tests against a node", ArgsUsage: " ", @@ -54,7 +53,7 @@ var ( testTAPFlag, }, } - rlpxSnapTestCommand = cli.Command{ + rlpxSnapTestCommand = &cli.Command{ Name: "snap-test", Usage: "Runs tests against a node", ArgsUsage: " ", @@ -106,16 +105,11 @@ func rlpxEthTest(ctx *cli.Context) error { if ctx.NArg() < 3 { exit("missing path to chain.rlp as command-line argument") } - suite, err := ethtest.NewSuite(getNodeArg(ctx), ctx.Args()[1], ctx.Args()[2]) + suite, err := ethtest.NewSuite(getNodeArg(ctx), ctx.Args().Get(1), ctx.Args().Get(2)) if err != nil { exit(err) } - // check if given node supports eth66, and if so, run eth66 protocol tests as well - is66Failed, _ := utesting.Run(utesting.Test{Name: "Is_66", Fn: suite.Is_66}) - if is66Failed { - return runTests(ctx, suite.EthTests()) - } - return runTests(ctx, suite.AllEthTests()) + return runTests(ctx, suite.EthTests()) } // rlpxSnapTest runs the snap protocol test suite. @@ -123,7 +117,7 @@ func rlpxSnapTest(ctx *cli.Context) error { if ctx.NArg() < 3 { exit("missing path to chain.rlp as command-line argument") } - suite, err := ethtest.NewSuite(getNodeArg(ctx), ctx.Args()[1], ctx.Args()[2]) + suite, err := ethtest.NewSuite(getNodeArg(ctx), ctx.Args().Get(1), ctx.Args().Get(2)) if err != nil { exit(err) } diff --git a/cmd/devp2p/runtest.go b/cmd/devp2p/runtest.go index 4168f8555bfbd..f72aa91119c5f 100644 --- a/cmd/devp2p/runtest.go +++ b/cmd/devp2p/runtest.go @@ -22,25 +22,25 @@ import ( "github.com/ethereum/go-ethereum/cmd/devp2p/internal/v4test" "github.com/ethereum/go-ethereum/internal/utesting" "github.com/ethereum/go-ethereum/log" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) var ( - testPatternFlag = cli.StringFlag{ + testPatternFlag = &cli.StringFlag{ Name: "run", Usage: "Pattern of test suite(s) to run", } - testTAPFlag = cli.BoolFlag{ + testTAPFlag = &cli.BoolFlag{ Name: "tap", Usage: "Output TAP", } // These two are specific to the discovery tests. - testListen1Flag = cli.StringFlag{ + testListen1Flag = &cli.StringFlag{ Name: "listen1", Usage: "IP address of the first tester", Value: v4test.Listen1, } - testListen2Flag = cli.StringFlag{ + testListen2Flag = &cli.StringFlag{ Name: "listen2", Usage: "IP address of the second tester", Value: v4test.Listen2, @@ -53,7 +53,7 @@ func runTests(ctx *cli.Context, tests []utesting.Test) error { tests = utesting.MatchTests(tests, ctx.String(testPatternFlag.Name)) } // Disable logging unless explicitly enabled. - if !ctx.GlobalIsSet("verbosity") && !ctx.GlobalIsSet("vmodule") { + if !ctx.IsSet("verbosity") && !ctx.IsSet("vmodule") { log.Root().SetHandler(log.DiscardHandler()) } // Run the tests. diff --git a/cmd/ethkey/changepassword.go b/cmd/ethkey/changepassword.go index bd8745f6db87b..4298e2b834079 100644 --- a/cmd/ethkey/changepassword.go +++ b/cmd/ethkey/changepassword.go @@ -23,15 +23,15 @@ import ( "github.com/ethereum/go-ethereum/accounts/keystore" "github.com/ethereum/go-ethereum/cmd/utils" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) -var newPassphraseFlag = cli.StringFlag{ +var newPassphraseFlag = &cli.StringFlag{ Name: "newpasswordfile", Usage: "the file that contains the new password for the keyfile", } -var commandChangePassphrase = cli.Command{ +var commandChangePassphrase = &cli.Command{ Name: "changepassword", Usage: "change the password on a keyfile", ArgsUsage: "", diff --git a/cmd/ethkey/generate.go b/cmd/ethkey/generate.go index 1b70b130bcd54..60d8b3c7795bf 100644 --- a/cmd/ethkey/generate.go +++ b/cmd/ethkey/generate.go @@ -26,7 +26,7 @@ import ( "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/crypto" "github.com/google/uuid" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) type outputGenerate struct { @@ -35,17 +35,17 @@ type outputGenerate struct { } var ( - privateKeyFlag = cli.StringFlag{ + privateKeyFlag = &cli.StringFlag{ Name: "privatekey", Usage: "file containing a raw private key to encrypt", } - lightKDFFlag = cli.BoolFlag{ + lightKDFFlag = &cli.BoolFlag{ Name: "lightkdf", Usage: "use less secure scrypt parameters", } ) -var commandGenerate = cli.Command{ +var commandGenerate = &cli.Command{ Name: "generate", Usage: "generate new keyfile", ArgsUsage: "[ ]", diff --git a/cmd/ethkey/inspect.go b/cmd/ethkey/inspect.go index efcaecd389d3b..29b1c13e859b9 100644 --- a/cmd/ethkey/inspect.go +++ b/cmd/ethkey/inspect.go @@ -24,7 +24,7 @@ import ( "github.com/ethereum/go-ethereum/accounts/keystore" "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/crypto" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) type outputInspect struct { @@ -34,13 +34,13 @@ type outputInspect struct { } var ( - privateFlag = cli.BoolFlag{ + privateFlag = &cli.BoolFlag{ Name: "private", Usage: "include the private key in the output", } ) -var commandInspect = cli.Command{ +var commandInspect = &cli.Command{ Name: "inspect", Usage: "inspect a keyfile", ArgsUsage: "", diff --git a/cmd/ethkey/main.go b/cmd/ethkey/main.go index 6db39174c4615..2b21f3cd70a4e 100644 --- a/cmd/ethkey/main.go +++ b/cmd/ethkey/main.go @@ -21,7 +21,7 @@ import ( "os" "github.com/ethereum/go-ethereum/internal/flags" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) const ( @@ -36,23 +36,22 @@ var app *cli.App func init() { app = flags.NewApp(gitCommit, gitDate, "an Ethereum key manager") - app.Commands = []cli.Command{ + app.Commands = []*cli.Command{ commandGenerate, commandInspect, commandChangePassphrase, commandSignMessage, commandVerifyMessage, } - cli.CommandHelpTemplate = flags.OriginCommandHelpTemplate } // Commonly used command line flags. var ( - passphraseFlag = cli.StringFlag{ + passphraseFlag = &cli.StringFlag{ Name: "passwordfile", Usage: "the file that contains the password for the keyfile", } - jsonFlag = cli.BoolFlag{ + jsonFlag = &cli.BoolFlag{ Name: "json", Usage: "output JSON instead of human-readable format", } diff --git a/cmd/ethkey/message.go b/cmd/ethkey/message.go index 1a58eeb536fa5..6b8dec03cd670 100644 --- a/cmd/ethkey/message.go +++ b/cmd/ethkey/message.go @@ -21,23 +21,24 @@ import ( "fmt" "os" + "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/accounts/keystore" "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) type outputSign struct { Signature string } -var msgfileFlag = cli.StringFlag{ +var msgfileFlag = &cli.StringFlag{ Name: "msgfile", Usage: "file containing the message to sign/verify", } -var commandSignMessage = cli.Command{ +var commandSignMessage = &cli.Command{ Name: "signmessage", Usage: "sign a message", ArgsUsage: " ", @@ -68,7 +69,7 @@ To sign a message contained in a file, use the --msgfile flag. utils.Fatalf("Error decrypting key: %v", err) } - signature, err := crypto.Sign(signHash(message), key.PrivateKey) + signature, err := crypto.Sign(accounts.TextHash(message), key.PrivateKey) if err != nil { utils.Fatalf("Failed to sign message: %v", err) } @@ -88,7 +89,7 @@ type outputVerify struct { RecoveredPublicKey string } -var commandVerifyMessage = cli.Command{ +var commandVerifyMessage = &cli.Command{ Name: "verifymessage", Usage: "verify the signature of a signed message", ArgsUsage: "
", @@ -113,7 +114,7 @@ It is possible to refer to a file containing the message.`, utils.Fatalf("Signature encoding is not hexadecimal: %v", err) } - recoveredPubkey, err := crypto.SigToPub(signHash(message), signature) + recoveredPubkey, err := crypto.SigToPub(accounts.TextHash(message), signature) if err != nil || recoveredPubkey == nil { utils.Fatalf("Signature verification failed: %v", err) } @@ -143,7 +144,7 @@ It is possible to refer to a file containing the message.`, func getMessage(ctx *cli.Context, msgarg int) []byte { if file := ctx.String(msgfileFlag.Name); file != "" { - if len(ctx.Args()) > msgarg { + if ctx.NArg() > msgarg { utils.Fatalf("Can't use --msgfile and message argument at the same time.") } msg, err := os.ReadFile(file) @@ -151,9 +152,9 @@ func getMessage(ctx *cli.Context, msgarg int) []byte { utils.Fatalf("Can't read message file: %v", err) } return msg - } else if len(ctx.Args()) == msgarg+1 { + } else if ctx.NArg() == msgarg+1 { return []byte(ctx.Args().Get(msgarg)) } - utils.Fatalf("Invalid number of arguments: want %d, got %d", msgarg+1, len(ctx.Args())) + utils.Fatalf("Invalid number of arguments: want %d, got %d", msgarg+1, ctx.NArg()) return nil } diff --git a/cmd/ethkey/utils.go b/cmd/ethkey/utils.go index b81e70913b5b4..2821145089ec6 100644 --- a/cmd/ethkey/utils.go +++ b/cmd/ethkey/utils.go @@ -23,8 +23,7 @@ import ( "strings" "github.com/ethereum/go-ethereum/cmd/utils" - "github.com/ethereum/go-ethereum/crypto" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) // getPassphrase obtains a passphrase given by the user. It first checks the @@ -46,18 +45,6 @@ func getPassphrase(ctx *cli.Context, confirmation bool) string { return utils.GetPassPhrase("", confirmation) } -// signHash is a helper function that calculates a hash for the given message -// that can be safely used to calculate a signature from. -// -// The hash is calculated as -// keccak256("\x19Ethereum Signed Message:\n"${message length}${message}). -// -// This gives context to the signed message and prevents signing of transactions. -func signHash(data []byte) []byte { - msg := fmt.Sprintf("\x19Ethereum Signed Message:\n%d%s", len(data), data) - return crypto.Keccak256([]byte(msg)) -} - // mustPrintJSON prints the JSON encoding of the given object and // exits the program with an error message when the marshaling fails. func mustPrintJSON(jsonObject interface{}) { diff --git a/cmd/evm/compiler.go b/cmd/evm/compiler.go index 880f995f057cd..699d434bb0e13 100644 --- a/cmd/evm/compiler.go +++ b/cmd/evm/compiler.go @@ -23,10 +23,10 @@ import ( "github.com/ethereum/go-ethereum/cmd/evm/internal/compiler" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) -var compileCommand = cli.Command{ +var compileCommand = &cli.Command{ Action: compileCmd, Name: "compile", Usage: "compiles easm source to evm binary", @@ -34,7 +34,7 @@ var compileCommand = cli.Command{ } func compileCmd(ctx *cli.Context) error { - debug := ctx.GlobalBool(DebugFlag.Name) + debug := ctx.Bool(DebugFlag.Name) if len(ctx.Args().First()) == 0 { return errors.New("filename required") diff --git a/cmd/evm/disasm.go b/cmd/evm/disasm.go index 918b01376791a..a6a16fd13b77c 100644 --- a/cmd/evm/disasm.go +++ b/cmd/evm/disasm.go @@ -23,10 +23,10 @@ import ( "strings" "github.com/ethereum/go-ethereum/core/asm" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) -var disasmCommand = cli.Command{ +var disasmCommand = &cli.Command{ Action: disasmCmd, Name: "disasm", Usage: "disassembles evm binary", @@ -43,8 +43,8 @@ func disasmCmd(ctx *cli.Context) error { return err } in = string(input) - case ctx.GlobalIsSet(InputFlag.Name): - in = ctx.GlobalString(InputFlag.Name) + case ctx.IsSet(InputFlag.Name): + in = ctx.String(InputFlag.Name) default: return errors.New("missing filename or --input value") } diff --git a/cmd/evm/internal/t8ntool/block.go b/cmd/evm/internal/t8ntool/block.go index 9839afd5f488b..4a070b6c71b54 100644 --- a/cmd/evm/internal/t8ntool/block.go +++ b/cmd/evm/internal/t8ntool/block.go @@ -33,7 +33,7 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) //go:generate go run github.com/fjl/gencodec -type header -field-override headerMarshaling -out gen_header.go diff --git a/cmd/evm/internal/t8ntool/execution.go b/cmd/evm/internal/t8ntool/execution.go index 83a0025344a48..77f6ec37158be 100644 --- a/cmd/evm/internal/t8ntool/execution.go +++ b/cmd/evm/internal/t8ntool/execution.go @@ -100,7 +100,6 @@ type rejectedTx struct { func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, txs types.Transactions, miningReward int64, getTracerFn func(txIndex int, txHash common.Hash) (tracer vm.EVMLogger, err error)) (*state.StateDB, *ExecutionResult, error) { - // Capture errors for BLOCKHASH operation, if we haven't been supplied the // required blockhashes var hashError error @@ -241,7 +240,7 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, minerReward.Add(minerReward, perOmmer) // Add (8-delta)/8 reward := big.NewInt(8) - reward.Sub(reward, big.NewInt(0).SetUint64(ommer.Delta)) + reward.Sub(reward, new(big.Int).SetUint64(ommer.Delta)) reward.Mul(reward, blockReward) reward.Div(reward, big.NewInt(8)) statedb.AddBalance(ommer.Address, reward) @@ -269,7 +268,7 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, } func MakePreState(db ethdb.Database, accounts core.GenesisAlloc) *state.StateDB { - sdb := state.NewDatabase(db) + sdb := state.NewDatabaseWithConfig(db, &trie.Config{Preimages: true}) statedb, _ := state.New(common.Hash{}, sdb, nil) for addr, a := range accounts { statedb.SetCode(addr, a.Code) diff --git a/cmd/evm/internal/t8ntool/flags.go b/cmd/evm/internal/t8ntool/flags.go index de666f1151231..626220315e19e 100644 --- a/cmd/evm/internal/t8ntool/flags.go +++ b/cmd/evm/internal/t8ntool/flags.go @@ -22,45 +22,47 @@ import ( "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/tests" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) var ( - TraceFlag = cli.BoolFlag{ + TraceFlag = &cli.BoolFlag{ Name: "trace", Usage: "Output full trace logs to files .jsonl", } - TraceDisableMemoryFlag = cli.BoolTFlag{ + TraceDisableMemoryFlag = &cli.BoolFlag{ Name: "trace.nomemory", + Value: true, Usage: "Disable full memory dump in traces (deprecated)", } - TraceEnableMemoryFlag = cli.BoolFlag{ + TraceEnableMemoryFlag = &cli.BoolFlag{ Name: "trace.memory", Usage: "Enable full memory dump in traces", } - TraceDisableStackFlag = cli.BoolFlag{ + TraceDisableStackFlag = &cli.BoolFlag{ Name: "trace.nostack", Usage: "Disable stack output in traces", } - TraceDisableReturnDataFlag = cli.BoolTFlag{ + TraceDisableReturnDataFlag = &cli.BoolFlag{ Name: "trace.noreturndata", + Value: true, Usage: "Disable return data output in traces (deprecated)", } - TraceEnableReturnDataFlag = cli.BoolFlag{ + TraceEnableReturnDataFlag = &cli.BoolFlag{ Name: "trace.returndata", Usage: "Enable return data output in traces", } - OutputBasedir = cli.StringFlag{ + OutputBasedir = &cli.StringFlag{ Name: "output.basedir", Usage: "Specifies where output files are placed. Will be created if it does not exist.", Value: "", } - OutputBodyFlag = cli.StringFlag{ + OutputBodyFlag = &cli.StringFlag{ Name: "output.body", Usage: "If set, the RLP of the transactions (block body) will be written to this file.", Value: "", } - OutputAllocFlag = cli.StringFlag{ + OutputAllocFlag = &cli.StringFlag{ Name: "output.alloc", Usage: "Determines where to put the `alloc` of the post-state.\n" + "\t`stdout` - into the stdout output\n" + @@ -68,7 +70,7 @@ var ( "\t - into the file ", Value: "alloc.json", } - OutputResultFlag = cli.StringFlag{ + OutputResultFlag = &cli.StringFlag{ Name: "output.result", Usage: "Determines where to put the `result` (stateroot, txroot etc) of the post-state.\n" + "\t`stdout` - into the stdout output\n" + @@ -76,7 +78,7 @@ var ( "\t - into the file ", Value: "result.json", } - OutputBlockFlag = cli.StringFlag{ + OutputBlockFlag = &cli.StringFlag{ Name: "output.block", Usage: "Determines where to put the `block` after building.\n" + "\t`stdout` - into the stdout output\n" + @@ -84,65 +86,65 @@ var ( "\t - into the file ", Value: "block.json", } - InputAllocFlag = cli.StringFlag{ + InputAllocFlag = &cli.StringFlag{ Name: "input.alloc", Usage: "`stdin` or file name of where to find the prestate alloc to use.", Value: "alloc.json", } - InputEnvFlag = cli.StringFlag{ + InputEnvFlag = &cli.StringFlag{ Name: "input.env", Usage: "`stdin` or file name of where to find the prestate env to use.", Value: "env.json", } - InputTxsFlag = cli.StringFlag{ + InputTxsFlag = &cli.StringFlag{ Name: "input.txs", Usage: "`stdin` or file name of where to find the transactions to apply. " + "If the file extension is '.rlp', then the data is interpreted as an RLP list of signed transactions." + "The '.rlp' format is identical to the output.body format.", Value: "txs.json", } - InputHeaderFlag = cli.StringFlag{ + InputHeaderFlag = &cli.StringFlag{ Name: "input.header", Usage: "`stdin` or file name of where to find the block header to use.", Value: "header.json", } - InputOmmersFlag = cli.StringFlag{ + InputOmmersFlag = &cli.StringFlag{ Name: "input.ommers", Usage: "`stdin` or file name of where to find the list of ommer header RLPs to use.", } - InputTxsRlpFlag = cli.StringFlag{ + InputTxsRlpFlag = &cli.StringFlag{ Name: "input.txs", Usage: "`stdin` or file name of where to find the transactions list in RLP form.", Value: "txs.rlp", } - SealCliqueFlag = cli.StringFlag{ + SealCliqueFlag = &cli.StringFlag{ Name: "seal.clique", Usage: "Seal block with Clique. `stdin` or file name of where to find the Clique sealing data.", } - SealEthashFlag = cli.BoolFlag{ + SealEthashFlag = &cli.BoolFlag{ Name: "seal.ethash", Usage: "Seal block with ethash.", } - SealEthashDirFlag = cli.StringFlag{ + SealEthashDirFlag = &cli.StringFlag{ Name: "seal.ethash.dir", Usage: "Path to ethash DAG. If none exists, a new DAG will be generated.", } - SealEthashModeFlag = cli.StringFlag{ + SealEthashModeFlag = &cli.StringFlag{ Name: "seal.ethash.mode", Usage: "Defines the type and amount of PoW verification an ethash engine makes.", Value: "normal", } - RewardFlag = cli.Int64Flag{ + RewardFlag = &cli.Int64Flag{ Name: "state.reward", Usage: "Mining reward. Set to -1 to disable", Value: 0, } - ChainIDFlag = cli.Int64Flag{ + ChainIDFlag = &cli.Int64Flag{ Name: "state.chainid", Usage: "ChainID to use", Value: 1, } - ForknameFlag = cli.StringFlag{ + ForknameFlag = &cli.StringFlag{ Name: "state.fork", Usage: fmt.Sprintf("Name of ruleset to use."+ "\n\tAvailable forknames:"+ @@ -152,9 +154,9 @@ var ( "\n\tSyntax (+ExtraEip)", strings.Join(tests.AvailableForks(), "\n\t "), strings.Join(vm.ActivateableEips(), ", ")), - Value: "ArrowGlacier", + Value: "GrayGlacier", } - VerbosityFlag = cli.IntFlag{ + VerbosityFlag = &cli.IntFlag{ Name: "verbosity", Usage: "sets the verbosity level", Value: 3, diff --git a/cmd/evm/internal/t8ntool/transaction.go b/cmd/evm/internal/t8ntool/transaction.go index 6f1c964ada026..3409c0a3bf01f 100644 --- a/cmd/evm/internal/t8ntool/transaction.go +++ b/cmd/evm/internal/t8ntool/transaction.go @@ -32,7 +32,7 @@ import ( "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/tests" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) type result struct { diff --git a/cmd/evm/internal/t8ntool/transition.go b/cmd/evm/internal/t8ntool/transition.go index b254baa995827..73a28e91c097f 100644 --- a/cmd/evm/internal/t8ntool/transition.go +++ b/cmd/evm/internal/t8ntool/transition.go @@ -38,7 +38,7 @@ import ( "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/tests" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) const ( diff --git a/cmd/evm/internal/t8ntool/utils.go b/cmd/evm/internal/t8ntool/utils.go index 1c54f09bf4176..8ec38c7618de0 100644 --- a/cmd/evm/internal/t8ntool/utils.go +++ b/cmd/evm/internal/t8ntool/utils.go @@ -21,7 +21,7 @@ import ( "fmt" "os" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) // readFile reads the json-data in the provided path and marshals into dest. diff --git a/cmd/evm/main.go b/cmd/evm/main.go index 2f404d48e9031..994b6436ad94d 100644 --- a/cmd/evm/main.go +++ b/cmd/evm/main.go @@ -23,115 +23,118 @@ import ( "os" "github.com/ethereum/go-ethereum/cmd/evm/internal/t8ntool" - "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/internal/flags" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) -var gitCommit = "" // Git SHA1 commit hash of the release (set via linker flags) -var gitDate = "" - var ( + gitCommit = "" // Git SHA1 commit hash of the release (set via linker flags) + gitDate = "" + app = flags.NewApp(gitCommit, gitDate, "the evm command line interface") +) - DebugFlag = cli.BoolFlag{ +var ( + DebugFlag = &cli.BoolFlag{ Name: "debug", Usage: "output full trace logs", } - MemProfileFlag = cli.StringFlag{ + MemProfileFlag = &cli.StringFlag{ Name: "memprofile", Usage: "creates a memory profile at the given path", } - CPUProfileFlag = cli.StringFlag{ + CPUProfileFlag = &cli.StringFlag{ Name: "cpuprofile", Usage: "creates a CPU profile at the given path", } - StatDumpFlag = cli.BoolFlag{ + StatDumpFlag = &cli.BoolFlag{ Name: "statdump", Usage: "displays stack and heap memory information", } - CodeFlag = cli.StringFlag{ + CodeFlag = &cli.StringFlag{ Name: "code", Usage: "EVM code", } - CodeFileFlag = cli.StringFlag{ + CodeFileFlag = &cli.StringFlag{ Name: "codefile", Usage: "File containing EVM code. If '-' is specified, code is read from stdin ", } - GasFlag = cli.Uint64Flag{ + GasFlag = &cli.Uint64Flag{ Name: "gas", Usage: "gas limit for the evm", Value: 10000000000, } - PriceFlag = utils.BigFlag{ + PriceFlag = &flags.BigFlag{ Name: "price", Usage: "price set for the evm", Value: new(big.Int), } - ValueFlag = utils.BigFlag{ + ValueFlag = &flags.BigFlag{ Name: "value", Usage: "value set for the evm", Value: new(big.Int), } - DumpFlag = cli.BoolFlag{ + DumpFlag = &cli.BoolFlag{ Name: "dump", Usage: "dumps the state after the run", } - InputFlag = cli.StringFlag{ + InputFlag = &cli.StringFlag{ Name: "input", Usage: "input for the EVM", } - InputFileFlag = cli.StringFlag{ + InputFileFlag = &cli.StringFlag{ Name: "inputfile", Usage: "file containing input for the EVM", } - VerbosityFlag = cli.IntFlag{ + VerbosityFlag = &cli.IntFlag{ Name: "verbosity", Usage: "sets the verbosity level", } - BenchFlag = cli.BoolFlag{ + BenchFlag = &cli.BoolFlag{ Name: "bench", Usage: "benchmark the execution", } - CreateFlag = cli.BoolFlag{ + CreateFlag = &cli.BoolFlag{ Name: "create", Usage: "indicates the action should be create rather than call", } - GenesisFlag = cli.StringFlag{ + GenesisFlag = &cli.StringFlag{ Name: "prestate", Usage: "JSON file with prestate (genesis) config", } - MachineFlag = cli.BoolFlag{ + MachineFlag = &cli.BoolFlag{ Name: "json", Usage: "output trace logs in machine readable format (json)", } - SenderFlag = cli.StringFlag{ + SenderFlag = &cli.StringFlag{ Name: "sender", Usage: "The transaction origin", } - ReceiverFlag = cli.StringFlag{ + ReceiverFlag = &cli.StringFlag{ Name: "receiver", Usage: "The transaction receiver (execution context)", } - DisableMemoryFlag = cli.BoolTFlag{ + DisableMemoryFlag = &cli.BoolFlag{ Name: "nomemory", + Value: true, Usage: "disable memory output", } - DisableStackFlag = cli.BoolFlag{ + DisableStackFlag = &cli.BoolFlag{ Name: "nostack", Usage: "disable stack output", } - DisableStorageFlag = cli.BoolFlag{ + DisableStorageFlag = &cli.BoolFlag{ Name: "nostorage", Usage: "disable storage output", } - DisableReturnDataFlag = cli.BoolTFlag{ + DisableReturnDataFlag = &cli.BoolFlag{ Name: "noreturndata", + Value: true, Usage: "enable return data output", } ) -var stateTransitionCommand = cli.Command{ +var stateTransitionCommand = &cli.Command{ Name: "transition", Aliases: []string{"t8n"}, Usage: "executes a full state transition", @@ -156,7 +159,8 @@ var stateTransitionCommand = cli.Command{ t8ntool.VerbosityFlag, }, } -var transactionCommand = cli.Command{ + +var transactionCommand = &cli.Command{ Name: "transaction", Aliases: []string{"t9n"}, Usage: "performs transaction validation", @@ -169,7 +173,7 @@ var transactionCommand = cli.Command{ }, } -var blockBuilderCommand = cli.Command{ +var blockBuilderCommand = &cli.Command{ Name: "block-builder", Aliases: []string{"b11r"}, Usage: "builds a block", @@ -214,7 +218,7 @@ func init() { DisableStorageFlag, DisableReturnDataFlag, } - app.Commands = []cli.Command{ + app.Commands = []*cli.Command{ compileCommand, disasmCommand, runCommand, @@ -223,7 +227,6 @@ func init() { transactionCommand, blockBuilderCommand, } - cli.CommandHelpTemplate = flags.OriginCommandHelpTemplate } func main() { diff --git a/cmd/evm/runner.go b/cmd/evm/runner.go index 5680c07a40ee4..9b1975c0500ef 100644 --- a/cmd/evm/runner.go +++ b/cmd/evm/runner.go @@ -37,12 +37,13 @@ import ( "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/core/vm/runtime" "github.com/ethereum/go-ethereum/eth/tracers/logger" + "github.com/ethereum/go-ethereum/internal/flags" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) -var runCommand = cli.Command{ +var runCommand = &cli.Command{ Action: runCmd, Name: "run", Usage: "run arbitrary evm binary", @@ -106,14 +107,14 @@ func timedExec(bench bool, execFunc func() ([]byte, uint64, error)) (output []by func runCmd(ctx *cli.Context) error { glogger := log.NewGlogHandler(log.StreamHandler(os.Stderr, log.TerminalFormat(false))) - glogger.Verbosity(log.Lvl(ctx.GlobalInt(VerbosityFlag.Name))) + glogger.Verbosity(log.Lvl(ctx.Int(VerbosityFlag.Name))) log.Root().SetHandler(glogger) logconfig := &logger.Config{ - EnableMemory: !ctx.GlobalBool(DisableMemoryFlag.Name), - DisableStack: ctx.GlobalBool(DisableStackFlag.Name), - DisableStorage: ctx.GlobalBool(DisableStorageFlag.Name), - EnableReturnData: !ctx.GlobalBool(DisableReturnDataFlag.Name), - Debug: ctx.GlobalBool(DebugFlag.Name), + EnableMemory: !ctx.Bool(DisableMemoryFlag.Name), + DisableStack: ctx.Bool(DisableStackFlag.Name), + DisableStorage: ctx.Bool(DisableStorageFlag.Name), + EnableReturnData: !ctx.Bool(DisableReturnDataFlag.Name), + Debug: ctx.Bool(DebugFlag.Name), } var ( @@ -125,37 +126,37 @@ func runCmd(ctx *cli.Context) error { receiver = common.BytesToAddress([]byte("receiver")) genesisConfig *core.Genesis ) - if ctx.GlobalBool(MachineFlag.Name) { + if ctx.Bool(MachineFlag.Name) { tracer = logger.NewJSONLogger(logconfig, os.Stdout) - } else if ctx.GlobalBool(DebugFlag.Name) { + } else if ctx.Bool(DebugFlag.Name) { debugLogger = logger.NewStructLogger(logconfig) tracer = debugLogger } else { debugLogger = logger.NewStructLogger(logconfig) } - if ctx.GlobalString(GenesisFlag.Name) != "" { - gen := readGenesis(ctx.GlobalString(GenesisFlag.Name)) + if ctx.String(GenesisFlag.Name) != "" { + gen := readGenesis(ctx.String(GenesisFlag.Name)) genesisConfig = gen db := rawdb.NewMemoryDatabase() - genesis := gen.ToBlock(db) + genesis := gen.MustCommit(db) statedb, _ = state.New(genesis.Root(), state.NewDatabase(db), nil) chainConfig = gen.Config } else { statedb, _ = state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) genesisConfig = new(core.Genesis) } - if ctx.GlobalString(SenderFlag.Name) != "" { - sender = common.HexToAddress(ctx.GlobalString(SenderFlag.Name)) + if ctx.String(SenderFlag.Name) != "" { + sender = common.HexToAddress(ctx.String(SenderFlag.Name)) } statedb.CreateAccount(sender) - if ctx.GlobalString(ReceiverFlag.Name) != "" { - receiver = common.HexToAddress(ctx.GlobalString(ReceiverFlag.Name)) + if ctx.String(ReceiverFlag.Name) != "" { + receiver = common.HexToAddress(ctx.String(ReceiverFlag.Name)) } var code []byte - codeFileFlag := ctx.GlobalString(CodeFileFlag.Name) - codeFlag := ctx.GlobalString(CodeFlag.Name) + codeFileFlag := ctx.String(CodeFileFlag.Name) + codeFlag := ctx.String(CodeFlag.Name) // The '--code' or '--codefile' flag overrides code in state if codeFileFlag != "" || codeFlag != "" { @@ -197,7 +198,7 @@ func runCmd(ctx *cli.Context) error { } code = common.Hex2Bytes(bin) } - initialGas := ctx.GlobalUint64(GasFlag.Name) + initialGas := ctx.Uint64(GasFlag.Name) if genesisConfig.GasLimit != 0 { initialGas = genesisConfig.GasLimit } @@ -205,19 +206,19 @@ func runCmd(ctx *cli.Context) error { Origin: sender, State: statedb, GasLimit: initialGas, - GasPrice: utils.GlobalBig(ctx, PriceFlag.Name), - Value: utils.GlobalBig(ctx, ValueFlag.Name), + GasPrice: flags.GlobalBig(ctx, PriceFlag.Name), + Value: flags.GlobalBig(ctx, ValueFlag.Name), Difficulty: genesisConfig.Difficulty, Time: new(big.Int).SetUint64(genesisConfig.Timestamp), Coinbase: genesisConfig.Coinbase, BlockNumber: new(big.Int).SetUint64(genesisConfig.Number), EVMConfig: vm.Config{ Tracer: tracer, - Debug: ctx.GlobalBool(DebugFlag.Name) || ctx.GlobalBool(MachineFlag.Name), + Debug: ctx.Bool(DebugFlag.Name) || ctx.Bool(MachineFlag.Name), }, } - if cpuProfilePath := ctx.GlobalString(CPUProfileFlag.Name); cpuProfilePath != "" { + if cpuProfilePath := ctx.String(CPUProfileFlag.Name); cpuProfilePath != "" { f, err := os.Create(cpuProfilePath) if err != nil { fmt.Println("could not create CPU profile: ", err) @@ -237,14 +238,14 @@ func runCmd(ctx *cli.Context) error { } var hexInput []byte - if inputFileFlag := ctx.GlobalString(InputFileFlag.Name); inputFileFlag != "" { + if inputFileFlag := ctx.String(InputFileFlag.Name); inputFileFlag != "" { var err error if hexInput, err = os.ReadFile(inputFileFlag); err != nil { fmt.Printf("could not load input from file: %v\n", err) os.Exit(1) } } else { - hexInput = []byte(ctx.GlobalString(InputFlag.Name)) + hexInput = []byte(ctx.String(InputFlag.Name)) } hexInput = bytes.TrimSpace(hexInput) if len(hexInput)%2 != 0 { @@ -254,7 +255,7 @@ func runCmd(ctx *cli.Context) error { input := common.FromHex(string(hexInput)) var execFunc func() ([]byte, uint64, error) - if ctx.GlobalBool(CreateFlag.Name) { + if ctx.Bool(CreateFlag.Name) { input = append(code, input...) execFunc = func() ([]byte, uint64, error) { output, _, gasLeft, err := runtime.Create(input, &runtimeConfig) @@ -269,16 +270,16 @@ func runCmd(ctx *cli.Context) error { } } - bench := ctx.GlobalBool(BenchFlag.Name) + bench := ctx.Bool(BenchFlag.Name) output, leftOverGas, stats, err := timedExec(bench, execFunc) - if ctx.GlobalBool(DumpFlag.Name) { + if ctx.Bool(DumpFlag.Name) { statedb.Commit(true) statedb.IntermediateRoot(true) fmt.Println(string(statedb.Dump(nil))) } - if memProfilePath := ctx.GlobalString(MemProfileFlag.Name); memProfilePath != "" { + if memProfilePath := ctx.String(MemProfileFlag.Name); memProfilePath != "" { f, err := os.Create(memProfilePath) if err != nil { fmt.Println("could not create memory profile: ", err) @@ -291,7 +292,7 @@ func runCmd(ctx *cli.Context) error { f.Close() } - if ctx.GlobalBool(DebugFlag.Name) { + if ctx.Bool(DebugFlag.Name) { if debugLogger != nil { fmt.Fprintln(os.Stderr, "#### TRACE ####") logger.WriteTrace(os.Stderr, debugLogger.StructLogs()) @@ -300,7 +301,7 @@ func runCmd(ctx *cli.Context) error { logger.WriteLogs(os.Stderr, statedb.Logs()) } - if bench || ctx.GlobalBool(StatDumpFlag.Name) { + if bench || ctx.Bool(StatDumpFlag.Name) { fmt.Fprintf(os.Stderr, `EVM gas used: %d execution time: %v allocations: %d @@ -308,7 +309,7 @@ allocated bytes: %d `, initialGas-leftOverGas, stats.time, stats.allocs, stats.bytesAllocated) } if tracer == nil { - fmt.Printf("0x%x\n", output) + fmt.Printf("%#x\n", output) if err != nil { fmt.Printf(" error: %v\n", err) } diff --git a/cmd/evm/staterunner.go b/cmd/evm/staterunner.go index fcdac33eedfb9..36f4e19b0bea6 100644 --- a/cmd/evm/staterunner.go +++ b/cmd/evm/staterunner.go @@ -28,10 +28,10 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/tests" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) -var stateTestCommand = cli.Command{ +var stateTestCommand = &cli.Command{ Action: stateTestCmd, Name: "statetest", Usage: "executes the given state tests", @@ -54,25 +54,25 @@ func stateTestCmd(ctx *cli.Context) error { } // Configure the go-ethereum logger glogger := log.NewGlogHandler(log.StreamHandler(os.Stderr, log.TerminalFormat(false))) - glogger.Verbosity(log.Lvl(ctx.GlobalInt(VerbosityFlag.Name))) + glogger.Verbosity(log.Lvl(ctx.Int(VerbosityFlag.Name))) log.Root().SetHandler(glogger) // Configure the EVM logger config := &logger.Config{ - EnableMemory: !ctx.GlobalBool(DisableMemoryFlag.Name), - DisableStack: ctx.GlobalBool(DisableStackFlag.Name), - DisableStorage: ctx.GlobalBool(DisableStorageFlag.Name), - EnableReturnData: !ctx.GlobalBool(DisableReturnDataFlag.Name), + EnableMemory: !ctx.Bool(DisableMemoryFlag.Name), + DisableStack: ctx.Bool(DisableStackFlag.Name), + DisableStorage: ctx.Bool(DisableStorageFlag.Name), + EnableReturnData: !ctx.Bool(DisableReturnDataFlag.Name), } var ( tracer vm.EVMLogger debugger *logger.StructLogger ) switch { - case ctx.GlobalBool(MachineFlag.Name): + case ctx.Bool(MachineFlag.Name): tracer = logger.NewJSONLogger(config, os.Stderr) - case ctx.GlobalBool(DebugFlag.Name): + case ctx.Bool(DebugFlag.Name): debugger = logger.NewStructLogger(config) tracer = debugger @@ -91,7 +91,7 @@ func stateTestCmd(ctx *cli.Context) error { // Iterate over all the tests, run them and aggregate the results cfg := vm.Config{ Tracer: tracer, - Debug: ctx.GlobalBool(DebugFlag.Name) || ctx.GlobalBool(MachineFlag.Name), + Debug: ctx.Bool(DebugFlag.Name) || ctx.Bool(MachineFlag.Name), } results := make([]StatetestResult, 0, len(tests)) for key, test := range tests { @@ -100,13 +100,13 @@ func stateTestCmd(ctx *cli.Context) error { result := &StatetestResult{Name: key, Fork: st.Fork, Pass: true} _, s, err := test.Run(st, cfg, false) // print state root for evmlab tracing - if ctx.GlobalBool(MachineFlag.Name) && s != nil { + if ctx.Bool(MachineFlag.Name) && s != nil { fmt.Fprintf(os.Stderr, "{\"stateRoot\": \"%x\"}\n", s.IntermediateRoot(false)) } if err != nil { // Test failed, mark as so and dump any state to aid debugging result.Pass, result.Error = false, err.Error() - if ctx.GlobalBool(DumpFlag.Name) && s != nil { + if ctx.Bool(DumpFlag.Name) && s != nil { dump := s.RawDump(nil) result.State = &dump } @@ -115,7 +115,7 @@ func stateTestCmd(ctx *cli.Context) error { results = append(results, *result) // Print any structured logs collected - if ctx.GlobalBool(DebugFlag.Name) { + if ctx.Bool(DebugFlag.Name) { if debugger != nil { fmt.Fprintln(os.Stderr, "#### TRACE ####") logger.WriteTrace(os.Stderr, debugger.StructLogs()) diff --git a/cmd/evm/t8n_test.go b/cmd/evm/t8n_test.go index 92c01398ba364..72c062e8d9236 100644 --- a/cmd/evm/t8n_test.go +++ b/cmd/evm/t8n_test.go @@ -211,6 +211,14 @@ func TestT8n(t *testing.T) { output: t8nOutput{result: true}, expOut: "exp_arrowglacier.json", }, + { // Difficulty calculation on gray glacier + base: "./testdata/19", + input: t8nInput{ + "alloc.json", "txs.json", "env.json", "GrayGlacier", "", + }, + output: t8nOutput{result: true}, + expOut: "exp_grayglacier.json", + }, { // Sign unprotected (pre-EIP155) transaction base: "./testdata/23", input: t8nInput{ @@ -236,7 +244,6 @@ func TestT8n(t *testing.T) { expExitCode: 3, }, } { - args := []string{"t8n"} args = append(args, tc.output.get()...) args = append(args, tc.input.get(tc.base)...) @@ -347,7 +354,6 @@ func TestT9n(t *testing.T) { expExitCode: t8ntool.ErrorIO, }, } { - args := []string{"t9n"} args = append(args, tc.input.get(tc.base)...) @@ -467,7 +473,6 @@ func TestB11r(t *testing.T) { expOut: "exp.json", }, } { - args := []string{"b11r"} args = append(args, tc.input.get(tc.base)...) diff --git a/cmd/evm/testdata/19/exp_grayglacier.json b/cmd/evm/testdata/19/exp_grayglacier.json new file mode 100644 index 0000000000000..95a3cb1685cfe --- /dev/null +++ b/cmd/evm/testdata/19/exp_grayglacier.json @@ -0,0 +1,12 @@ +{ + "result": { + "stateRoot": "0x6f058887ca01549716789c380ede95aecc510e6d1fdc4dbf67d053c7c07f4bdc", + "txRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "receiptsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "logsHash": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "receipts": [], + "currentDifficulty": "0x2000000004000", + "gasUsed": "0x0" + } +} \ No newline at end of file diff --git a/cmd/evm/testdata/19/readme.md b/cmd/evm/testdata/19/readme.md index 5fae183f48860..095d4525d4fe9 100644 --- a/cmd/evm/testdata/19/readme.md +++ b/cmd/evm/testdata/19/readme.md @@ -1,9 +1,9 @@ ## Difficulty calculation This test shows how the `evm t8n` can be used to calculate the (ethash) difficulty, if none is provided by the caller, -this time on `ArrowGlacier` (Eip 4345). +this time on `GrayGlacier` (Eip 5133). -Calculating it (with an empty set of txs) using `ArrowGlacier` rules (and no provided unclehash for the parent block): +Calculating it (with an empty set of txs) using `GrayGlacier` rules (and no provided unclehash for the parent block): ``` -[user@work evm]$ ./evm t8n --input.alloc=./testdata/14/alloc.json --input.txs=./testdata/14/txs.json --input.env=./testdata/14/env.json --output.result=stdout --state.fork=ArrowGlacier +[user@work evm]$ ./evm t8n --input.alloc=./testdata/19/alloc.json --input.txs=./testdata/19/txs.json --input.env=./testdata/19/env.json --output.result=stdout --state.fork=GrayGlacier ``` \ No newline at end of file diff --git a/cmd/faucet/README.md b/cmd/faucet/README.md index f27e94aa9e72c..c31676b0dac19 100644 --- a/cmd/faucet/README.md +++ b/cmd/faucet/README.md @@ -10,9 +10,10 @@ The `faucet` is a single binary app (everything included) with all configuration First thing's first, the `faucet` needs to connect to an Ethereum network, for which it needs the necessary genesis and network infos. Each of the following flags must be set: -- `-genesis` is a path to a file containin the network `genesis.json`. or using: +- `-genesis` is a path to a file containing the network `genesis.json`. or using: - `-goerli` with the faucet with Görli network config - `-rinkeby` with the faucet with Rinkeby network config + - `-sepolia` with the faucet with Sepolia network config - `-network` is the devp2p network id used during connection - `-bootnodes` is a list of `enode://` ids to join the network through diff --git a/cmd/faucet/faucet.go b/cmd/faucet/faucet.go index bcb837062f6fd..dfb7d326dc49d 100644 --- a/cmd/faucet/faucet.go +++ b/cmd/faucet/faucet.go @@ -86,6 +86,7 @@ var ( goerliFlag = flag.Bool("goerli", false, "Initializes the faucet with Görli network config") rinkebyFlag = flag.Bool("rinkeby", false, "Initializes the faucet with Rinkeby network config") + sepoliaFlag = flag.Bool("sepolia", false, "Initializes the faucet with Sepolia network config") ) var ( @@ -143,7 +144,7 @@ func main() { log.Crit("Failed to render the faucet template", "err", err) } // Load and parse the genesis block requested by the user - genesis, err := getGenesis(*genesisFlag, *goerliFlag, *rinkebyFlag) + genesis, err := getGenesis(*genesisFlag, *goerliFlag, *rinkebyFlag, *sepoliaFlag) if err != nil { log.Crit("Failed to parse genesis config", "err", err) } @@ -247,7 +248,7 @@ func newFaucet(genesis *core.Genesis, port int, enodes []*enode.Node, network ui cfg.SyncMode = downloader.LightSync cfg.NetworkId = network cfg.Genesis = genesis - utils.SetDNSDiscoveryDefaults(&cfg, genesis.ToBlock(nil).Hash()) + utils.SetDNSDiscoveryDefaults(&cfg, genesis.ToBlock().Hash()) lesBackend, err := les.New(stack, &cfg) if err != nil { @@ -708,7 +709,7 @@ func authTwitter(url string, tokenV1, tokenV2 string) (string, string, string, c case tokenV2 != "": return authTwitterWithTokenV2(tweetID, tokenV2) } - // Twiter API token isn't provided so we just load the public posts + // Twitter API token isn't provided so we just load the public posts // and scrape it for the Ethereum address and profile URL. We need to load // the mobile page though since the main page loads tweet contents via JS. url = strings.Replace(url, "https://twitter.com/", "https://mobile.twitter.com/", 1) @@ -860,7 +861,7 @@ func authFacebook(url string) (string, string, common.Address, error) { address := common.HexToAddress(string(regexp.MustCompile("0x[0-9a-fA-F]{40}").Find(body))) if address == (common.Address{}) { //lint:ignore ST1005 This error is to be displayed in the browser - return "", "", common.Address{}, errors.New("No Ethereum address found to fund") + return "", "", common.Address{}, errors.New("No Ethereum address found to fund. Please check the post URL and verify that it can be viewed publicly.") } var avatar string if parts = regexp.MustCompile(`src="([^"]+fbcdn\.net[^"]+)"`).FindStringSubmatch(string(body)); len(parts) == 2 { @@ -882,7 +883,7 @@ func authNoAuth(url string) (string, string, common.Address, error) { } // getGenesis returns a genesis based on input args -func getGenesis(genesisFlag string, goerliFlag bool, rinkebyFlag bool) (*core.Genesis, error) { +func getGenesis(genesisFlag string, goerliFlag bool, rinkebyFlag bool, sepoliaFlag bool) (*core.Genesis, error) { switch { case genesisFlag != "": var genesis core.Genesis @@ -892,6 +893,8 @@ func getGenesis(genesisFlag string, goerliFlag bool, rinkebyFlag bool) (*core.Ge return core.DefaultGoerliGenesisBlock(), nil case rinkebyFlag: return core.DefaultRinkebyGenesisBlock(), nil + case sepoliaFlag: + return core.DefaultSepoliaGenesisBlock(), nil default: return nil, fmt.Errorf("no genesis flag provided") } diff --git a/cmd/geth/accountcmd.go b/cmd/geth/accountcmd.go index 0b7d58e8888cd..5158b7606cdef 100644 --- a/cmd/geth/accountcmd.go +++ b/cmd/geth/accountcmd.go @@ -25,29 +25,27 @@ import ( "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) var ( - walletCommand = cli.Command{ + walletCommand = &cli.Command{ Name: "wallet", Usage: "Manage Ethereum presale wallets", ArgsUsage: "", - Category: "ACCOUNT COMMANDS", Description: ` geth wallet import /path/to/my/presale.wallet will prompt for your password and imports your ether presale account. It can be used non-interactively with the --password option taking a passwordfile as argument containing the wallet password in plaintext.`, - Subcommands: []cli.Command{ + Subcommands: []*cli.Command{ { Name: "import", Usage: "Import Ethereum presale wallet", ArgsUsage: "", - Action: utils.MigrateFlags(importWallet), - Category: "ACCOUNT COMMANDS", + Action: importWallet, Flags: []cli.Flag{ utils.DataDirFlag, utils.KeyStoreDirFlag, @@ -64,10 +62,9 @@ passwordfile as argument containing the wallet password in plaintext.`, }, } - accountCommand = cli.Command{ - Name: "account", - Usage: "Manage accounts", - Category: "ACCOUNT COMMANDS", + accountCommand = &cli.Command{ + Name: "account", + Usage: "Manage accounts", Description: ` Manage accounts, list all existing accounts, import a private key into a new @@ -88,11 +85,11 @@ It is safe to transfer the entire directory or the individual keys therein between ethereum nodes by simply copying. Make sure you backup your keys regularly.`, - Subcommands: []cli.Command{ + Subcommands: []*cli.Command{ { Name: "list", Usage: "Print summary of existing accounts", - Action: utils.MigrateFlags(accountList), + Action: accountList, Flags: []cli.Flag{ utils.DataDirFlag, utils.KeyStoreDirFlag, @@ -103,7 +100,7 @@ Print a short summary of all accounts`, { Name: "new", Usage: "Create a new account", - Action: utils.MigrateFlags(accountCreate), + Action: accountCreate, Flags: []cli.Flag{ utils.DataDirFlag, utils.KeyStoreDirFlag, @@ -128,7 +125,7 @@ password to file or expose in any other way. { Name: "update", Usage: "Update an existing account", - Action: utils.MigrateFlags(accountUpdate), + Action: accountUpdate, ArgsUsage: "
", Flags: []cli.Flag{ utils.DataDirFlag, @@ -157,7 +154,7 @@ changing your password is only possible interactively. { Name: "import", Usage: "Import a private key into a new account", - Action: utils.MigrateFlags(accountImport), + Action: accountImport, Flags: []cli.Flag{ utils.DataDirFlag, utils.KeyStoreDirFlag, @@ -239,14 +236,15 @@ func ambiguousAddrRecovery(ks *keystore.KeyStore, err *keystore.AmbiguousAddrErr } fmt.Println("Testing your password against all of them...") var match *accounts.Account - for _, a := range err.Matches { - if err := ks.Unlock(a, auth); err == nil { - match = &a + for i, a := range err.Matches { + if e := ks.Unlock(a, auth); e == nil { + match = &err.Matches[i] break } } if match == nil { utils.Fatalf("None of the listed files could be unlocked.") + return accounts.Account{} } fmt.Printf("Your password unlocked %s\n", match.URL) fmt.Println("In order to avoid this warning, you need to remove the following duplicate key files:") @@ -262,7 +260,7 @@ func ambiguousAddrRecovery(ks *keystore.KeyStore, err *keystore.AmbiguousAddrErr func accountCreate(ctx *cli.Context) error { cfg := gethConfig{Node: defaultNodeConfig()} // Load config file. - if file := ctx.GlobalString(configFileFlag.Name); file != "" { + if file := ctx.String(configFileFlag.Name); file != "" { if err := loadConfig(file, &cfg); err != nil { utils.Fatalf("%v", err) } @@ -299,13 +297,13 @@ func accountCreate(ctx *cli.Context) error { // accountUpdate transitions an account from a previous format to the current // one, also providing the possibility to change the pass-phrase. func accountUpdate(ctx *cli.Context) error { - if len(ctx.Args()) == 0 { + if ctx.Args().Len() == 0 { utils.Fatalf("No accounts specified to update") } stack, _ := makeConfigNode(ctx) ks := stack.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore) - for _, addr := range ctx.Args() { + for _, addr := range ctx.Args().Slice() { account, oldPassword := unlockAccount(ks, addr, 0, nil) newPassword := utils.GetPassPhraseWithList("Please give a new password. Do not forget this password.", true, 0, nil) if err := ks.Update(account, oldPassword, newPassword); err != nil { @@ -316,10 +314,10 @@ func accountUpdate(ctx *cli.Context) error { } func importWallet(ctx *cli.Context) error { - keyfile := ctx.Args().First() - if len(keyfile) == 0 { - utils.Fatalf("keyfile must be given as argument") + if ctx.Args().Len() != 1 { + utils.Fatalf("keyfile must be given as the only argument") } + keyfile := ctx.Args().First() keyJSON, err := os.ReadFile(keyfile) if err != nil { utils.Fatalf("Could not read wallet file: %v", err) @@ -338,10 +336,10 @@ func importWallet(ctx *cli.Context) error { } func accountImport(ctx *cli.Context) error { - keyfile := ctx.Args().First() - if len(keyfile) == 0 { - utils.Fatalf("keyfile must be given as argument") + if ctx.Args().Len() != 1 { + utils.Fatalf("keyfile must be given as the only argument") } + keyfile := ctx.Args().First() key, err := crypto.LoadECDSA(keyfile) if err != nil { utils.Fatalf("Failed to load the private key: %v", err) diff --git a/cmd/geth/accountcmd_test.go b/cmd/geth/accountcmd_test.go index 0c22e8c9bf576..84b9c33c24cf7 100644 --- a/cmd/geth/accountcmd_test.go +++ b/cmd/geth/accountcmd_test.go @@ -49,20 +49,27 @@ func TestAccountListEmpty(t *testing.T) { func TestAccountList(t *testing.T) { datadir := tmpDatadirWithKeystore(t) - geth := runGeth(t, "account", "list", "--datadir", datadir) - defer geth.ExpectExit() + var want = ` +Account #0: {7ef5a6135f1fd6a02593eedc869c6d41d934aef8} keystore://{{.Datadir}}/keystore/UTC--2016-03-22T12-57-55.920751759Z--7ef5a6135f1fd6a02593eedc869c6d41d934aef8 +Account #1: {f466859ead1932d743d622cb74fc058882e8648a} keystore://{{.Datadir}}/keystore/aaa +Account #2: {289d485d9771714cce91d3393d764e1311907acc} keystore://{{.Datadir}}/keystore/zzz +` if runtime.GOOS == "windows" { - geth.Expect(` + want = ` Account #0: {7ef5a6135f1fd6a02593eedc869c6d41d934aef8} keystore://{{.Datadir}}\keystore\UTC--2016-03-22T12-57-55.920751759Z--7ef5a6135f1fd6a02593eedc869c6d41d934aef8 Account #1: {f466859ead1932d743d622cb74fc058882e8648a} keystore://{{.Datadir}}\keystore\aaa Account #2: {289d485d9771714cce91d3393d764e1311907acc} keystore://{{.Datadir}}\keystore\zzz -`) - } else { - geth.Expect(` -Account #0: {7ef5a6135f1fd6a02593eedc869c6d41d934aef8} keystore://{{.Datadir}}/keystore/UTC--2016-03-22T12-57-55.920751759Z--7ef5a6135f1fd6a02593eedc869c6d41d934aef8 -Account #1: {f466859ead1932d743d622cb74fc058882e8648a} keystore://{{.Datadir}}/keystore/aaa -Account #2: {289d485d9771714cce91d3393d764e1311907acc} keystore://{{.Datadir}}/keystore/zzz -`) +` + } + { + geth := runGeth(t, "account", "list", "--datadir", datadir) + geth.Expect(want) + geth.ExpectExit() + } + { + geth := runGeth(t, "--datadir", datadir, "account", "list") + geth.Expect(want) + geth.ExpectExit() } } @@ -110,6 +117,20 @@ func TestAccountImport(t *testing.T) { } } +func TestAccountHelp(t *testing.T) { + geth := runGeth(t, "account", "-h") + geth.WaitExit() + if have, want := geth.ExitStatus(), 0; have != want { + t.Errorf("exit error, have %d want %d", have, want) + } + + geth = runGeth(t, "account", "import", "-h") + geth.WaitExit() + if have, want := geth.ExitStatus(), 0; have != want { + t.Errorf("exit error, have %d want %d", have, want) + } +} + func importAccountWithExpect(t *testing.T, key string, expected string) { dir := t.TempDir() keyfile := filepath.Join(dir, "key.prv") @@ -120,7 +141,7 @@ func importAccountWithExpect(t *testing.T, key string, expected string) { if err := os.WriteFile(passwordFile, []byte("foobar"), 0600); err != nil { t.Error(err) } - geth := runGeth(t, "--lightkdf", "account", "import", keyfile, "-password", passwordFile) + geth := runGeth(t, "--lightkdf", "account", "import", "-password", passwordFile, keyfile) defer geth.ExpectExit() geth.Expect(expected) } @@ -180,11 +201,12 @@ Fatal: could not decrypt key with given password func TestUnlockFlag(t *testing.T) { geth := runMinimalGeth(t, "--port", "0", "--ipcdisable", "--datadir", tmpDatadirWithKeystore(t), - "--unlock", "f466859ead1932d743d622cb74fc058882e8648a", "js", "testdata/empty.js") + "--unlock", "f466859ead1932d743d622cb74fc058882e8648a", "console", "--exec", "loadScript('testdata/empty.js')") geth.Expect(` Unlocking account f466859ead1932d743d622cb74fc058882e8648a | Attempt 1/3 !! Unsupported terminal, password will be echoed. Password: {{.InputLine "foobar"}} +undefined `) geth.ExpectExit() @@ -201,7 +223,7 @@ Password: {{.InputLine "foobar"}} func TestUnlockFlagWrongPassword(t *testing.T) { geth := runMinimalGeth(t, "--port", "0", "--ipcdisable", "--datadir", tmpDatadirWithKeystore(t), - "--unlock", "f466859ead1932d743d622cb74fc058882e8648a", "js", "testdata/empty.js") + "--unlock", "f466859ead1932d743d622cb74fc058882e8648a", "console", "--exec", "loadScript('testdata/empty.js')") defer geth.ExpectExit() geth.Expect(` @@ -219,7 +241,7 @@ Fatal: Failed to unlock account f466859ead1932d743d622cb74fc058882e8648a (could // https://github.com/ethereum/go-ethereum/issues/1785 func TestUnlockFlagMultiIndex(t *testing.T) { geth := runMinimalGeth(t, "--port", "0", "--ipcdisable", "--datadir", tmpDatadirWithKeystore(t), - "--unlock", "f466859ead1932d743d622cb74fc058882e8648a", "--unlock", "0,2", "js", "testdata/empty.js") + "--unlock", "f466859ead1932d743d622cb74fc058882e8648a", "--unlock", "0,2", "console", "--exec", "loadScript('testdata/empty.js')") geth.Expect(` Unlocking account 0 | Attempt 1/3 @@ -227,6 +249,7 @@ Unlocking account 0 | Attempt 1/3 Password: {{.InputLine "foobar"}} Unlocking account 2 | Attempt 1/3 Password: {{.InputLine "foobar"}} +undefined `) geth.ExpectExit() @@ -244,8 +267,11 @@ Password: {{.InputLine "foobar"}} func TestUnlockFlagPasswordFile(t *testing.T) { geth := runMinimalGeth(t, "--port", "0", "--ipcdisable", "--datadir", tmpDatadirWithKeystore(t), - "--unlock", "f466859ead1932d743d622cb74fc058882e8648a", "--password", "testdata/passwords.txt", "--unlock", "0,2", "js", "testdata/empty.js") + "--unlock", "f466859ead1932d743d622cb74fc058882e8648a", "--password", "testdata/passwords.txt", "--unlock", "0,2", "console", "--exec", "loadScript('testdata/empty.js')") + geth.Expect(` +undefined +`) geth.ExpectExit() wantMessages := []string{ @@ -275,7 +301,7 @@ func TestUnlockFlagAmbiguous(t *testing.T) { geth := runMinimalGeth(t, "--port", "0", "--ipcdisable", "--datadir", tmpDatadirWithKeystore(t), "--unlock", "f466859ead1932d743d622cb74fc058882e8648a", "--keystore", store, "--unlock", "f466859ead1932d743d622cb74fc058882e8648a", - "js", "testdata/empty.js") + "console", "--exec", "loadScript('testdata/empty.js')") defer geth.ExpectExit() // Helper for the expect template, returns absolute keystore path. @@ -294,6 +320,7 @@ Testing your password against all of them... Your password unlocked keystore://{{keypath "1"}} In order to avoid this warning, you need to remove the following duplicate key files: keystore://{{keypath "2"}} +undefined `) geth.ExpectExit() diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index 6e19edeb46ebe..a3016c4b091f9 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -35,20 +35,20 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/internal/flags" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/node" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) var ( - initCommand = cli.Command{ - Action: utils.MigrateFlags(initGenesis), + initCommand = &cli.Command{ + Action: initGenesis, Name: "init", Usage: "Bootstrap and initialize a new genesis block", ArgsUsage: "", Flags: utils.DatabasePathFlags, - Category: "BLOCKCHAIN COMMANDS", Description: ` The init command initializes a new genesis block and definition for the network. This is a destructive action and changes the network in which you will be @@ -56,22 +56,21 @@ participating. It expects the genesis file as argument.`, } - dumpGenesisCommand = cli.Command{ - Action: utils.MigrateFlags(dumpGenesis), + dumpGenesisCommand = &cli.Command{ + Action: dumpGenesis, Name: "dumpgenesis", Usage: "Dumps genesis block JSON configuration to stdout", ArgsUsage: "", Flags: utils.NetworkFlags, - Category: "BLOCKCHAIN COMMANDS", Description: ` The dumpgenesis command dumps the genesis block configuration in JSON format to stdout.`, } - importCommand = cli.Command{ - Action: utils.MigrateFlags(importChain), + importCommand = &cli.Command{ + Action: importChain, Name: "import", Usage: "Import a blockchain file", ArgsUsage: " ( ... ) ", - Flags: append([]cli.Flag{ + Flags: flags.Merge([]cli.Flag{ utils.CacheFlag, utils.SyncModeFlag, utils.GCModeFlag, @@ -93,8 +92,7 @@ The dumpgenesis command dumps the genesis block configuration in JSON format to utils.MetricsInfluxDBBucketFlag, utils.MetricsInfluxDBOrganizationFlag, utils.TxLookupLimitFlag, - }, utils.DatabasePathFlags...), - Category: "BLOCKCHAIN COMMANDS", + }, utils.DatabasePathFlags), Description: ` The import command imports blocks from an RLP-encoded form. The form can be one file with several RLP-encoded blocks, or several files can be used. @@ -102,16 +100,15 @@ with several RLP-encoded blocks, or several files can be used. If only one file is used, import error will result in failure. If several files are used, processing will proceed even if an individual RLP-file import failure occurs.`, } - exportCommand = cli.Command{ - Action: utils.MigrateFlags(exportChain), + exportCommand = &cli.Command{ + Action: exportChain, Name: "export", Usage: "Export blockchain into file", ArgsUsage: " [ ]", - Flags: append([]cli.Flag{ + Flags: flags.Merge([]cli.Flag{ utils.CacheFlag, utils.SyncModeFlag, - }, utils.DatabasePathFlags...), - Category: "BLOCKCHAIN COMMANDS", + }, utils.DatabasePathFlags), Description: ` Requires a first argument of the file to write to. Optional second and third arguments control the first and @@ -119,42 +116,40 @@ last block to write. In this mode, the file will be appended if already existing. If the file ends with .gz, the output will be gzipped.`, } - importPreimagesCommand = cli.Command{ - Action: utils.MigrateFlags(importPreimages), + importPreimagesCommand = &cli.Command{ + Action: importPreimages, Name: "import-preimages", Usage: "Import the preimage database from an RLP stream", ArgsUsage: "", - Flags: append([]cli.Flag{ + Flags: flags.Merge([]cli.Flag{ utils.CacheFlag, utils.SyncModeFlag, - }, utils.DatabasePathFlags...), - Category: "BLOCKCHAIN COMMANDS", + }, utils.DatabasePathFlags), Description: ` The import-preimages command imports hash preimages from an RLP encoded stream. It's deprecated, please use "geth db import" instead. `, } - exportPreimagesCommand = cli.Command{ - Action: utils.MigrateFlags(exportPreimages), + exportPreimagesCommand = &cli.Command{ + Action: exportPreimages, Name: "export-preimages", Usage: "Export the preimage database into an RLP stream", ArgsUsage: "", - Flags: append([]cli.Flag{ + Flags: flags.Merge([]cli.Flag{ utils.CacheFlag, utils.SyncModeFlag, - }, utils.DatabasePathFlags...), - Category: "BLOCKCHAIN COMMANDS", + }, utils.DatabasePathFlags), Description: ` The export-preimages command exports hash preimages to an RLP encoded stream. It's deprecated, please use "geth db export" instead. `, } - dumpCommand = cli.Command{ - Action: utils.MigrateFlags(dump), + dumpCommand = &cli.Command{ + Action: dump, Name: "dump", Usage: "Dump a specific block from storage", ArgsUsage: "[? | ]", - Flags: append([]cli.Flag{ + Flags: flags.Merge([]cli.Flag{ utils.CacheFlag, utils.IterativeOutputFlag, utils.ExcludeCodeFlag, @@ -162,8 +157,7 @@ It's deprecated, please use "geth db export" instead. utils.IncludeIncompletesFlag, utils.StartKeyFlag, utils.DumpLimitFlag, - }, utils.DatabasePathFlags...), - Category: "BLOCKCHAIN COMMANDS", + }, utils.DatabasePathFlags), Description: ` This command dumps out the state for a given block (or latest, if none provided). `, @@ -173,10 +167,12 @@ This command dumps out the state for a given block (or latest, if none provided) // initGenesis will initialise the given JSON format genesis file and writes it as // the zero'd block (i.e. genesis) or will fail hard if it can't succeed. func initGenesis(ctx *cli.Context) error { - // Make sure we have a valid genesis JSON + if ctx.Args().Len() != 1 { + utils.Fatalf("need genesis.json file as the only argument") + } genesisPath := ctx.Args().First() if len(genesisPath) == 0 { - utils.Fatalf("Must supply path to genesis JSON file") + utils.Fatalf("invalid path to genesis file") } file, err := os.Open(genesisPath) if err != nil { @@ -192,7 +188,7 @@ func initGenesis(ctx *cli.Context) error { stack, _ := makeConfigNode(ctx) defer stack.Close() for _, name := range []string{"chaindata", "lightchaindata"} { - chaindb, err := stack.OpenDatabaseWithFreezer(name, 0, 0, ctx.GlobalString(utils.AncientFlag.Name), "", false) + chaindb, err := stack.OpenDatabaseWithFreezer(name, 0, 0, ctx.String(utils.AncientFlag.Name), "", false) if err != nil { utils.Fatalf("Failed to open database: %v", err) } @@ -219,7 +215,7 @@ func dumpGenesis(ctx *cli.Context) error { } func importChain(ctx *cli.Context) error { - if len(ctx.Args()) < 1 { + if ctx.Args().Len() < 1 { utils.Fatalf("This command requires an argument.") } // Start metrics export if enabled @@ -253,13 +249,13 @@ func importChain(ctx *cli.Context) error { var importErr error - if len(ctx.Args()) == 1 { + if ctx.Args().Len() == 1 { if err := utils.ImportChain(chain, ctx.Args().First()); err != nil { importErr = err log.Error("Import error", "err", err) } } else { - for _, arg := range ctx.Args() { + for _, arg := range ctx.Args().Slice() { if err := utils.ImportChain(chain, arg); err != nil { importErr = err log.Error("Import error", "file", arg, "err", err) @@ -281,7 +277,7 @@ func importChain(ctx *cli.Context) error { fmt.Printf("Allocations: %.3f million\n", float64(mem.Mallocs)/1000000) fmt.Printf("GC pause: %v\n\n", time.Duration(mem.PauseTotalNs)) - if ctx.GlobalBool(utils.NoCompactionFlag.Name) { + if ctx.Bool(utils.NoCompactionFlag.Name) { return nil } @@ -298,7 +294,7 @@ func importChain(ctx *cli.Context) error { } func exportChain(ctx *cli.Context) error { - if len(ctx.Args()) < 1 { + if ctx.Args().Len() < 1 { utils.Fatalf("This command requires an argument.") } @@ -310,7 +306,7 @@ func exportChain(ctx *cli.Context) error { var err error fp := ctx.Args().First() - if len(ctx.Args()) < 3 { + if ctx.Args().Len() < 3 { err = utils.ExportChain(chain, fp) } else { // This can be improved to allow for numbers larger than 9223372036854775807 @@ -337,7 +333,7 @@ func exportChain(ctx *cli.Context) error { // importPreimages imports preimage data from the specified file. func importPreimages(ctx *cli.Context) error { - if len(ctx.Args()) < 1 { + if ctx.Args().Len() < 1 { utils.Fatalf("This command requires an argument.") } @@ -356,7 +352,7 @@ func importPreimages(ctx *cli.Context) error { // exportPreimages dumps the preimage data to specified json file in streaming way. func exportPreimages(ctx *cli.Context) error { - if len(ctx.Args()) < 1 { + if ctx.Args().Len() < 1 { utils.Fatalf("This command requires an argument.") } stack, _ := makeConfigNode(ctx) @@ -388,12 +384,12 @@ func parseDumpConfig(ctx *cli.Context, stack *node.Node) (*state.DumpConfig, eth return nil, nil, common.Hash{}, fmt.Errorf("block %x not found", hash) } } else { - number, err := strconv.Atoi(arg) + number, err := strconv.ParseUint(arg, 10, 64) if err != nil { return nil, nil, common.Hash{}, err } - if hash := rawdb.ReadCanonicalHash(db, uint64(number)); hash != (common.Hash{}) { - header = rawdb.ReadHeader(db, hash, uint64(number)) + if hash := rawdb.ReadCanonicalHash(db, number); hash != (common.Hash{}) { + header = rawdb.ReadHeader(db, hash, number) } else { return nil, nil, common.Hash{}, fmt.Errorf("header for block %d not found", number) } diff --git a/cmd/geth/config.go b/cmd/geth/config.go index bb003c4c1ae43..30565fda6185e 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -20,12 +20,11 @@ import ( "bufio" "errors" "fmt" - "math/big" "os" "reflect" "unicode" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" "github.com/ethereum/go-ethereum/accounts/external" "github.com/ethereum/go-ethereum/accounts/keystore" @@ -35,6 +34,7 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/internal/ethapi" + "github.com/ethereum/go-ethereum/internal/flags" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/node" @@ -43,19 +43,19 @@ import ( ) var ( - dumpConfigCommand = cli.Command{ - Action: utils.MigrateFlags(dumpConfig), + dumpConfigCommand = &cli.Command{ + Action: dumpConfig, Name: "dumpconfig", Usage: "Show configuration values", ArgsUsage: "", - Flags: utils.GroupFlags(nodeFlags, rpcFlags), - Category: "MISCELLANEOUS COMMANDS", + Flags: flags.Merge(nodeFlags, rpcFlags), Description: `The dumpconfig command shows configuration values.`, } - configFileFlag = cli.StringFlag{ - Name: "config", - Usage: "TOML configuration file", + configFileFlag = &cli.StringFlag{ + Name: "config", + Usage: "TOML configuration file", + Category: flags.EthCategory, } ) @@ -127,7 +127,7 @@ func makeConfigNode(ctx *cli.Context) (*node.Node, gethConfig) { } // Load config file. - if file := ctx.GlobalString(configFileFlag.Name); file != "" { + if file := ctx.String(configFileFlag.Name); file != "" { if err := loadConfig(file, &cfg); err != nil { utils.Fatalf("%v", err) } @@ -145,8 +145,8 @@ func makeConfigNode(ctx *cli.Context) (*node.Node, gethConfig) { } utils.SetEthConfig(ctx, stack, &cfg.Eth) - if ctx.GlobalIsSet(utils.EthStatsURLFlag.Name) { - cfg.Ethstats.URL = ctx.GlobalString(utils.EthStatsURLFlag.Name) + if ctx.IsSet(utils.EthStatsURLFlag.Name) { + cfg.Ethstats.URL = ctx.String(utils.EthStatsURLFlag.Name) } applyMetricConfig(ctx, &cfg) @@ -156,15 +156,18 @@ func makeConfigNode(ctx *cli.Context) (*node.Node, gethConfig) { // makeFullNode loads geth configuration and creates the Ethereum backend. func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) { stack, cfg := makeConfigNode(ctx) - if ctx.GlobalIsSet(utils.OverrideArrowGlacierFlag.Name) { - cfg.Eth.OverrideArrowGlacier = new(big.Int).SetUint64(ctx.GlobalUint64(utils.OverrideArrowGlacierFlag.Name)) + if ctx.IsSet(utils.OverrideTerminalTotalDifficulty.Name) { + cfg.Eth.OverrideTerminalTotalDifficulty = flags.GlobalBig(ctx, utils.OverrideTerminalTotalDifficulty.Name) } - if ctx.GlobalIsSet(utils.OverrideTerminalTotalDifficulty.Name) { - cfg.Eth.OverrideTerminalTotalDifficulty = utils.GlobalBig(ctx, utils.OverrideTerminalTotalDifficulty.Name) + if ctx.IsSet(utils.OverrideTerminalTotalDifficultyPassed.Name) { + override := ctx.Bool(utils.OverrideTerminalTotalDifficultyPassed.Name) + cfg.Eth.OverrideTerminalTotalDifficultyPassed = &override } + backend, eth := utils.RegisterEthService(stack, &cfg.Eth) + // Warn users to migrate if they have a legacy freezer format. - if eth != nil { + if eth != nil && !ctx.IsSet(utils.IgnoreLegacyReceiptsFlag.Name) { firstIdx := uint64(0) // Hack to speed up check for mainnet because we know // the first non-empty block. @@ -176,14 +179,19 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) { if err != nil { log.Error("Failed to check db for legacy receipts", "err", err) } else if isLegacy { - log.Warn("Database has receipts with a legacy format. Please run `geth db freezer-migrate`.") + stack.Close() + utils.Fatalf("Database has receipts with a legacy format. Please run `geth db freezer-migrate`.") } } - // Configure GraphQL if requested - if ctx.GlobalIsSet(utils.GraphQLEnabledFlag.Name) { - utils.RegisterGraphQLService(stack, backend, cfg.Node) + // Configure log filter RPC API. + filterSystem := utils.RegisterFilterAPI(stack, backend, &cfg.Eth) + + // Configure GraphQL if requested. + if ctx.IsSet(utils.GraphQLEnabledFlag.Name) { + utils.RegisterGraphQLService(stack, backend, filterSystem, &cfg.Node) } + // Add the Ethereum Stats daemon if requested. if cfg.Ethstats.URL != "" { utils.RegisterEthStatsService(stack, backend, cfg.Ethstats.URL) @@ -221,47 +229,47 @@ func dumpConfig(ctx *cli.Context) error { } func applyMetricConfig(ctx *cli.Context, cfg *gethConfig) { - if ctx.GlobalIsSet(utils.MetricsEnabledFlag.Name) { - cfg.Metrics.Enabled = ctx.GlobalBool(utils.MetricsEnabledFlag.Name) + if ctx.IsSet(utils.MetricsEnabledFlag.Name) { + cfg.Metrics.Enabled = ctx.Bool(utils.MetricsEnabledFlag.Name) } - if ctx.GlobalIsSet(utils.MetricsEnabledExpensiveFlag.Name) { - cfg.Metrics.EnabledExpensive = ctx.GlobalBool(utils.MetricsEnabledExpensiveFlag.Name) + if ctx.IsSet(utils.MetricsEnabledExpensiveFlag.Name) { + cfg.Metrics.EnabledExpensive = ctx.Bool(utils.MetricsEnabledExpensiveFlag.Name) } - if ctx.GlobalIsSet(utils.MetricsHTTPFlag.Name) { - cfg.Metrics.HTTP = ctx.GlobalString(utils.MetricsHTTPFlag.Name) + if ctx.IsSet(utils.MetricsHTTPFlag.Name) { + cfg.Metrics.HTTP = ctx.String(utils.MetricsHTTPFlag.Name) } - if ctx.GlobalIsSet(utils.MetricsPortFlag.Name) { - cfg.Metrics.Port = ctx.GlobalInt(utils.MetricsPortFlag.Name) + if ctx.IsSet(utils.MetricsPortFlag.Name) { + cfg.Metrics.Port = ctx.Int(utils.MetricsPortFlag.Name) } - if ctx.GlobalIsSet(utils.MetricsEnableInfluxDBFlag.Name) { - cfg.Metrics.EnableInfluxDB = ctx.GlobalBool(utils.MetricsEnableInfluxDBFlag.Name) + if ctx.IsSet(utils.MetricsEnableInfluxDBFlag.Name) { + cfg.Metrics.EnableInfluxDB = ctx.Bool(utils.MetricsEnableInfluxDBFlag.Name) } - if ctx.GlobalIsSet(utils.MetricsInfluxDBEndpointFlag.Name) { - cfg.Metrics.InfluxDBEndpoint = ctx.GlobalString(utils.MetricsInfluxDBEndpointFlag.Name) + if ctx.IsSet(utils.MetricsInfluxDBEndpointFlag.Name) { + cfg.Metrics.InfluxDBEndpoint = ctx.String(utils.MetricsInfluxDBEndpointFlag.Name) } - if ctx.GlobalIsSet(utils.MetricsInfluxDBDatabaseFlag.Name) { - cfg.Metrics.InfluxDBDatabase = ctx.GlobalString(utils.MetricsInfluxDBDatabaseFlag.Name) + if ctx.IsSet(utils.MetricsInfluxDBDatabaseFlag.Name) { + cfg.Metrics.InfluxDBDatabase = ctx.String(utils.MetricsInfluxDBDatabaseFlag.Name) } - if ctx.GlobalIsSet(utils.MetricsInfluxDBUsernameFlag.Name) { - cfg.Metrics.InfluxDBUsername = ctx.GlobalString(utils.MetricsInfluxDBUsernameFlag.Name) + if ctx.IsSet(utils.MetricsInfluxDBUsernameFlag.Name) { + cfg.Metrics.InfluxDBUsername = ctx.String(utils.MetricsInfluxDBUsernameFlag.Name) } - if ctx.GlobalIsSet(utils.MetricsInfluxDBPasswordFlag.Name) { - cfg.Metrics.InfluxDBPassword = ctx.GlobalString(utils.MetricsInfluxDBPasswordFlag.Name) + if ctx.IsSet(utils.MetricsInfluxDBPasswordFlag.Name) { + cfg.Metrics.InfluxDBPassword = ctx.String(utils.MetricsInfluxDBPasswordFlag.Name) } - if ctx.GlobalIsSet(utils.MetricsInfluxDBTagsFlag.Name) { - cfg.Metrics.InfluxDBTags = ctx.GlobalString(utils.MetricsInfluxDBTagsFlag.Name) + if ctx.IsSet(utils.MetricsInfluxDBTagsFlag.Name) { + cfg.Metrics.InfluxDBTags = ctx.String(utils.MetricsInfluxDBTagsFlag.Name) } - if ctx.GlobalIsSet(utils.MetricsEnableInfluxDBV2Flag.Name) { - cfg.Metrics.EnableInfluxDBV2 = ctx.GlobalBool(utils.MetricsEnableInfluxDBV2Flag.Name) + if ctx.IsSet(utils.MetricsEnableInfluxDBV2Flag.Name) { + cfg.Metrics.EnableInfluxDBV2 = ctx.Bool(utils.MetricsEnableInfluxDBV2Flag.Name) } - if ctx.GlobalIsSet(utils.MetricsInfluxDBTokenFlag.Name) { - cfg.Metrics.InfluxDBToken = ctx.GlobalString(utils.MetricsInfluxDBTokenFlag.Name) + if ctx.IsSet(utils.MetricsInfluxDBTokenFlag.Name) { + cfg.Metrics.InfluxDBToken = ctx.String(utils.MetricsInfluxDBTokenFlag.Name) } - if ctx.GlobalIsSet(utils.MetricsInfluxDBBucketFlag.Name) { - cfg.Metrics.InfluxDBBucket = ctx.GlobalString(utils.MetricsInfluxDBBucketFlag.Name) + if ctx.IsSet(utils.MetricsInfluxDBBucketFlag.Name) { + cfg.Metrics.InfluxDBBucket = ctx.String(utils.MetricsInfluxDBBucketFlag.Name) } - if ctx.GlobalIsSet(utils.MetricsInfluxDBOrganizationFlag.Name) { - cfg.Metrics.InfluxDBOrganization = ctx.GlobalString(utils.MetricsInfluxDBOrganizationFlag.Name) + if ctx.IsSet(utils.MetricsInfluxDBOrganizationFlag.Name) { + cfg.Metrics.InfluxDBOrganization = ctx.String(utils.MetricsInfluxDBOrganizationFlag.Name) } } diff --git a/cmd/geth/consolecmd.go b/cmd/geth/consolecmd.go index 5167f8536a277..87bbe24b977ac 100644 --- a/cmd/geth/consolecmd.go +++ b/cmd/geth/consolecmd.go @@ -18,39 +18,36 @@ package main import ( "fmt" - "path/filepath" "strings" "github.com/ethereum/go-ethereum/cmd/utils" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/console" + "github.com/ethereum/go-ethereum/internal/flags" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/rpc" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) var ( consoleFlags = []cli.Flag{utils.JSpathFlag, utils.ExecFlag, utils.PreloadJSFlag} - consoleCommand = cli.Command{ - Action: utils.MigrateFlags(localConsole), - Name: "console", - Usage: "Start an interactive JavaScript environment", - Flags: utils.GroupFlags(nodeFlags, rpcFlags, consoleFlags), - Category: "CONSOLE COMMANDS", + consoleCommand = &cli.Command{ + Action: localConsole, + Name: "console", + Usage: "Start an interactive JavaScript environment", + Flags: flags.Merge(nodeFlags, rpcFlags, consoleFlags), Description: ` The Geth console is an interactive shell for the JavaScript runtime environment which exposes a node admin interface as well as the Ðapp JavaScript API. See https://geth.ethereum.org/docs/interface/javascript-console.`, } - attachCommand = cli.Command{ - Action: utils.MigrateFlags(remoteConsole), + attachCommand = &cli.Command{ + Action: remoteConsole, Name: "attach", Usage: "Start an interactive JavaScript environment (connect to node)", ArgsUsage: "[endpoint]", - Flags: utils.GroupFlags([]cli.Flag{utils.DataDirFlag}, consoleFlags), - Category: "CONSOLE COMMANDS", + Flags: flags.Merge([]cli.Flag{utils.DataDirFlag}, consoleFlags), Description: ` The Geth console is an interactive shell for the JavaScript runtime environment which exposes a node admin interface as well as the Ðapp JavaScript API. @@ -58,13 +55,12 @@ See https://geth.ethereum.org/docs/interface/javascript-console. This command allows to open a console on a running geth node.`, } - javascriptCommand = cli.Command{ - Action: utils.MigrateFlags(ephemeralConsole), + javascriptCommand = &cli.Command{ + Action: ephemeralConsole, Name: "js", - Usage: "Execute the specified JavaScript files", + Usage: "(DEPRECATED) Execute the specified JavaScript files", ArgsUsage: " [jsfile...]", - Flags: utils.GroupFlags(nodeFlags, consoleFlags), - Category: "CONSOLE COMMANDS", + Flags: flags.Merge(nodeFlags, consoleFlags), Description: ` The JavaScript VM exposes a node admin interface as well as the Ðapp JavaScript API. See https://geth.ethereum.org/docs/interface/javascript-console`, @@ -87,7 +83,7 @@ func localConsole(ctx *cli.Context) error { } config := console.Config{ DataDir: utils.MakeDataDir(ctx), - DocRoot: ctx.GlobalString(utils.JSpathFlag.Name), + DocRoot: ctx.String(utils.JSpathFlag.Name), Client: client, Preload: utils.MakeConsolePreloads(ctx), } @@ -98,7 +94,7 @@ func localConsole(ctx *cli.Context) error { defer console.Stop(false) // If only a short execution was requested, evaluate and return. - if script := ctx.GlobalString(utils.ExecFlag.Name); script != "" { + if script := ctx.String(utils.ExecFlag.Name); script != "" { console.Evaluate(script) return nil } @@ -119,33 +115,15 @@ func localConsole(ctx *cli.Context) error { // remoteConsole will connect to a remote geth instance, attaching a JavaScript // console to it. func remoteConsole(ctx *cli.Context) error { + if ctx.Args().Len() > 1 { + utils.Fatalf("invalid command-line: too many arguments") + } + endpoint := ctx.Args().First() if endpoint == "" { - path := node.DefaultDataDir() - if ctx.GlobalIsSet(utils.DataDirFlag.Name) { - path = ctx.GlobalString(utils.DataDirFlag.Name) - } - if path != "" { - if ctx.GlobalBool(utils.RopstenFlag.Name) { - // Maintain compatibility with older Geth configurations storing the - // Ropsten database in `testnet` instead of `ropsten`. - legacyPath := filepath.Join(path, "testnet") - if common.FileExist(legacyPath) { - path = legacyPath - } else { - path = filepath.Join(path, "ropsten") - } - } else if ctx.GlobalBool(utils.RinkebyFlag.Name) { - path = filepath.Join(path, "rinkeby") - } else if ctx.GlobalBool(utils.GoerliFlag.Name) { - path = filepath.Join(path, "goerli") - } else if ctx.GlobalBool(utils.SepoliaFlag.Name) { - path = filepath.Join(path, "sepolia") - } else if ctx.GlobalBool(utils.KilnFlag.Name) { - path = filepath.Join(path, "kiln") - } - } - endpoint = fmt.Sprintf("%s/geth.ipc", path) + cfg := defaultNodeConfig() + utils.SetDataDir(ctx, &cfg) + endpoint = cfg.IPCEndpoint() } client, err := dialRPC(endpoint) if err != nil { @@ -153,7 +131,7 @@ func remoteConsole(ctx *cli.Context) error { } config := console.Config{ DataDir: utils.MakeDataDir(ctx), - DocRoot: ctx.GlobalString(utils.JSpathFlag.Name), + DocRoot: ctx.String(utils.JSpathFlag.Name), Client: client, Preload: utils.MakeConsolePreloads(ctx), } @@ -163,7 +141,7 @@ func remoteConsole(ctx *cli.Context) error { } defer console.Stop(false) - if script := ctx.GlobalString(utils.ExecFlag.Name); script != "" { + if script := ctx.String(utils.ExecFlag.Name); script != "" { console.Evaluate(script) return nil } @@ -174,6 +152,19 @@ func remoteConsole(ctx *cli.Context) error { return nil } +// ephemeralConsole starts a new geth node, attaches an ephemeral JavaScript +// console to it, executes each of the files specified as arguments and tears +// everything down. +func ephemeralConsole(ctx *cli.Context) error { + var b strings.Builder + for _, file := range ctx.Args().Slice() { + b.Write([]byte(fmt.Sprintf("loadScript('%s');", file))) + } + utils.Fatalf(`The "js" command is deprecated. Please use the following instead: +geth --exec "%s" console`, b.String()) + return nil +} + // dialRPC returns a RPC client which connects to the given endpoint. // The check for empty endpoint implements the defaulting logic // for "geth attach" with no argument. @@ -187,48 +178,3 @@ func dialRPC(endpoint string) (*rpc.Client, error) { } return rpc.Dial(endpoint) } - -// ephemeralConsole starts a new geth node, attaches an ephemeral JavaScript -// console to it, executes each of the files specified as arguments and tears -// everything down. -func ephemeralConsole(ctx *cli.Context) error { - // Create and start the node based on the CLI flags - stack, backend := makeFullNode(ctx) - startNode(ctx, stack, backend, false) - defer stack.Close() - - // Attach to the newly started node and start the JavaScript console - client, err := stack.Attach() - if err != nil { - return fmt.Errorf("Failed to attach to the inproc geth: %v", err) - } - config := console.Config{ - DataDir: utils.MakeDataDir(ctx), - DocRoot: ctx.GlobalString(utils.JSpathFlag.Name), - Client: client, - Preload: utils.MakeConsolePreloads(ctx), - } - - console, err := console.New(config) - if err != nil { - return fmt.Errorf("Failed to start the JavaScript console: %v", err) - } - defer console.Stop(false) - - // Interrupt the JS interpreter when node is stopped. - go func() { - stack.Wait() - console.Stop(false) - }() - - // Evaluate each of the specified JavaScript files. - for _, file := range ctx.Args() { - if err = console.Execute(file); err != nil { - return fmt.Errorf("Failed to execute %s: %v", file, err) - } - } - - // The main script is now done, but keep running timers/callbacks. - console.Stop(true) - return nil -} diff --git a/cmd/geth/consolecmd_test.go b/cmd/geth/consolecmd_test.go index f4e8bf490a32e..442b82df0b3ce 100644 --- a/cmd/geth/consolecmd_test.go +++ b/cmd/geth/consolecmd_test.go @@ -41,7 +41,7 @@ func runMinimalGeth(t *testing.T, args ...string) *testgeth { // --ropsten to make the 'writing genesis to disk' faster (no accounts) // --networkid=1337 to avoid cache bump // --syncmode=full to avoid allocating fast sync bloom - allArgs := []string{"--ropsten", "--networkid", "1337", "--syncmode=full", "--port", "0", + allArgs := []string{"--ropsten", "--networkid", "1337", "--authrpc.port", "0", "--syncmode=full", "--port", "0", "--nat", "none", "--nodiscover", "--maxpeers", "0", "--cache", "64", "--datadir.minfreedisk", "0"} return runGeth(t, append(allArgs, args...)...) @@ -155,7 +155,7 @@ To exit, press ctrl-d or type exit } // trulyRandInt generates a crypto random integer used by the console tests to -// not clash network ports with other tests running cocurrently. +// not clash network ports with other tests running concurrently. func trulyRandInt(lo, hi int) int { num, _ := rand.Int(rand.Reader, big.NewInt(int64(hi-lo))) return int(num.Int64()) + lo diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index ace2849c9f065..ab74277123d76 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -22,7 +22,6 @@ import ( "os" "os/signal" "path/filepath" - "sort" "strconv" "strings" "syscall" @@ -37,29 +36,28 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/internal/flags" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/trie" "github.com/olekukonko/tablewriter" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) var ( - removedbCommand = cli.Command{ - Action: utils.MigrateFlags(removeDB), + removedbCommand = &cli.Command{ + Action: removeDB, Name: "removedb", Usage: "Remove blockchain and state databases", ArgsUsage: "", Flags: utils.DatabasePathFlags, - Category: "DATABASE COMMANDS", Description: ` Remove blockchain and state databases`, } - dbCommand = cli.Command{ + dbCommand = &cli.Command{ Name: "db", Usage: "Low level database operations", ArgsUsage: "", - Category: "DATABASE COMMANDS", - Subcommands: []cli.Command{ + Subcommands: []*cli.Command{ dbInspectCmd, dbStatCmd, dbCompactCmd, @@ -75,39 +73,39 @@ Remove blockchain and state databases`, dbCheckStateContentCmd, }, } - dbInspectCmd = cli.Command{ - Action: utils.MigrateFlags(inspect), + dbInspectCmd = &cli.Command{ + Action: inspect, Name: "inspect", ArgsUsage: " ", - Flags: utils.GroupFlags([]cli.Flag{ + Flags: flags.Merge([]cli.Flag{ utils.SyncModeFlag, }, utils.NetworkFlags, utils.DatabasePathFlags), Usage: "Inspect the storage size for each type of data in the database", Description: `This commands iterates the entire database. If the optional 'prefix' and 'start' arguments are provided, then the iteration is limited to the given subset of data.`, } - dbCheckStateContentCmd = cli.Command{ - Action: utils.MigrateFlags(checkStateContent), + dbCheckStateContentCmd = &cli.Command{ + Action: checkStateContent, Name: "check-state-content", ArgsUsage: "", - Flags: utils.GroupFlags(utils.NetworkFlags, utils.DatabasePathFlags), + Flags: flags.Merge(utils.NetworkFlags, utils.DatabasePathFlags), Usage: "Verify that state data is cryptographically correct", Description: `This command iterates the entire database for 32-byte keys, looking for rlp-encoded trie nodes. For each trie node encountered, it checks that the key corresponds to the keccak256(value). If this is not true, this indicates a data corruption.`, } - dbStatCmd = cli.Command{ - Action: utils.MigrateFlags(dbStats), + dbStatCmd = &cli.Command{ + Action: dbStats, Name: "stats", Usage: "Print leveldb statistics", - Flags: utils.GroupFlags([]cli.Flag{ + Flags: flags.Merge([]cli.Flag{ utils.SyncModeFlag, }, utils.NetworkFlags, utils.DatabasePathFlags), } - dbCompactCmd = cli.Command{ - Action: utils.MigrateFlags(dbCompact), + dbCompactCmd = &cli.Command{ + Action: dbCompact, Name: "compact", Usage: "Compact leveldb database. WARNING: May take a very long time", - Flags: utils.GroupFlags([]cli.Flag{ + Flags: flags.Merge([]cli.Flag{ utils.SyncModeFlag, utils.CacheFlag, utils.CacheDatabaseFlag, @@ -116,93 +114,93 @@ a data corruption.`, WARNING: This operation may take a very long time to finish, and may cause database corruption if it is aborted during execution'!`, } - dbGetCmd = cli.Command{ - Action: utils.MigrateFlags(dbGet), + dbGetCmd = &cli.Command{ + Action: dbGet, Name: "get", Usage: "Show the value of a database key", ArgsUsage: "", - Flags: utils.GroupFlags([]cli.Flag{ + Flags: flags.Merge([]cli.Flag{ utils.SyncModeFlag, }, utils.NetworkFlags, utils.DatabasePathFlags), Description: "This command looks up the specified database key from the database.", } - dbDeleteCmd = cli.Command{ - Action: utils.MigrateFlags(dbDelete), + dbDeleteCmd = &cli.Command{ + Action: dbDelete, Name: "delete", Usage: "Delete a database key (WARNING: may corrupt your database)", ArgsUsage: "", - Flags: utils.GroupFlags([]cli.Flag{ + Flags: flags.Merge([]cli.Flag{ utils.SyncModeFlag, }, utils.NetworkFlags, utils.DatabasePathFlags), Description: `This command deletes the specified database key from the database. WARNING: This is a low-level operation which may cause database corruption!`, } - dbPutCmd = cli.Command{ - Action: utils.MigrateFlags(dbPut), + dbPutCmd = &cli.Command{ + Action: dbPut, Name: "put", Usage: "Set the value of a database key (WARNING: may corrupt your database)", ArgsUsage: " ", - Flags: utils.GroupFlags([]cli.Flag{ + Flags: flags.Merge([]cli.Flag{ utils.SyncModeFlag, }, utils.NetworkFlags, utils.DatabasePathFlags), Description: `This command sets a given database key to the given value. WARNING: This is a low-level operation which may cause database corruption!`, } - dbGetSlotsCmd = cli.Command{ - Action: utils.MigrateFlags(dbDumpTrie), + dbGetSlotsCmd = &cli.Command{ + Action: dbDumpTrie, Name: "dumptrie", Usage: "Show the storage key/values of a given storage trie", ArgsUsage: " ", - Flags: utils.GroupFlags([]cli.Flag{ + Flags: flags.Merge([]cli.Flag{ utils.SyncModeFlag, }, utils.NetworkFlags, utils.DatabasePathFlags), Description: "This command looks up the specified database key from the database.", } - dbDumpFreezerIndex = cli.Command{ - Action: utils.MigrateFlags(freezerInspect), + dbDumpFreezerIndex = &cli.Command{ + Action: freezerInspect, Name: "freezer-index", - Usage: "Dump out the index of a given freezer type", - ArgsUsage: " ", - Flags: utils.GroupFlags([]cli.Flag{ + Usage: "Dump out the index of a specific freezer table", + ArgsUsage: " ", + Flags: flags.Merge([]cli.Flag{ utils.SyncModeFlag, }, utils.NetworkFlags, utils.DatabasePathFlags), Description: "This command displays information about the freezer index.", } - dbImportCmd = cli.Command{ - Action: utils.MigrateFlags(importLDBdata), + dbImportCmd = &cli.Command{ + Action: importLDBdata, Name: "import", Usage: "Imports leveldb-data from an exported RLP dump.", ArgsUsage: " has .gz suffix, gzip compression will be used.", ArgsUsage: " ", - Flags: utils.GroupFlags([]cli.Flag{ + Flags: flags.Merge([]cli.Flag{ utils.SyncModeFlag, }, utils.NetworkFlags, utils.DatabasePathFlags), Description: "Exports the specified chain data to an RLP encoded stream, optionally gzip-compressed.", } - dbMetadataCmd = cli.Command{ - Action: utils.MigrateFlags(showMetaData), + dbMetadataCmd = &cli.Command{ + Action: showMetaData, Name: "metadata", Usage: "Shows metadata about the chain status.", - Flags: utils.GroupFlags([]cli.Flag{ + Flags: flags.Merge([]cli.Flag{ utils.SyncModeFlag, }, utils.NetworkFlags, utils.DatabasePathFlags), Description: "Shows metadata about the chain status.", } - dbMigrateFreezerCmd = cli.Command{ - Action: utils.MigrateFlags(freezerMigrate), + dbMigrateFreezerCmd = &cli.Command{ + Action: freezerMigrate, Name: "freezer-migrate", Usage: "Migrate legacy parts of the freezer. (WARNING: may take a long time)", ArgsUsage: "", - Flags: utils.GroupFlags([]cli.Flag{ + Flags: flags.Merge([]cli.Flag{ utils.SyncModeFlag, }, utils.NetworkFlags, utils.DatabasePathFlags), Description: `The freezer-migrate command checks your database for receipts in a legacy format and updates those. @@ -276,7 +274,7 @@ func inspect(ctx *cli.Context) error { start []byte ) if ctx.NArg() > 2 { - return fmt.Errorf("Max 2 arguments: %v", ctx.Command.ArgsUsage) + return fmt.Errorf("max 2 arguments: %v", ctx.Command.ArgsUsage) } if ctx.NArg() >= 1 { if d, err := hexutil.Decode(ctx.Args().Get(0)); err != nil { @@ -307,7 +305,7 @@ func checkStateContent(ctx *cli.Context) error { start []byte ) if ctx.NArg() > 1 { - return fmt.Errorf("Max 1 argument: %v", ctx.Command.ArgsUsage) + return fmt.Errorf("max 1 argument: %v", ctx.Command.ArgsUsage) } if ctx.NArg() > 0 { if d, err := hexutil.Decode(ctx.Args().First()); err != nil { @@ -332,16 +330,16 @@ func checkStateContent(ctx *cli.Context) error { ) for it.Next() { count++ - v := it.Value() k := it.Key() + v := it.Value() hasher.Reset() hasher.Write(v) hasher.Read(got) if !bytes.Equal(k, got) { errs++ - fmt.Printf("Error at 0x%x\n", k) - fmt.Printf(" Hash: 0x%x\n", got) - fmt.Printf(" Data: 0x%x\n", v) + fmt.Printf("Error at %#x\n", k) + fmt.Printf(" Hash: %#x\n", got) + fmt.Printf(" Data: %#x\n", v) } if time.Since(lastLog) > 8*time.Second { log.Info("Iterating the database", "at", fmt.Sprintf("%#x", k), "elapsed", common.PrettyDuration(time.Since(startTime))) @@ -418,7 +416,7 @@ func dbGet(ctx *cli.Context) error { data, err := db.Get(key) if err != nil { - log.Info("Get operation failed", "key", fmt.Sprintf("0x%#x", key), "error", err) + log.Info("Get operation failed", "key", fmt.Sprintf("%#x", key), "error", err) return err } fmt.Printf("key %#x: %#x\n", key, data) @@ -446,7 +444,7 @@ func dbDelete(ctx *cli.Context) error { fmt.Printf("Previous value: %#x\n", data) } if err = db.Delete(key); err != nil { - log.Info("Delete operation returned an error", "key", fmt.Sprintf("0x%#x", key), "error", err) + log.Info("Delete operation returned an error", "key", fmt.Sprintf("%#x", key), "error", err) return err } return nil @@ -519,7 +517,7 @@ func dbDumpTrie(ctx *cli.Context) error { return err } } - theTrie, err := trie.New(stRoot, trie.NewDatabase(db)) + theTrie, err := trie.New(common.Hash{}, stRoot, trie.NewDatabase(db)) if err != nil { return err } @@ -537,43 +535,35 @@ func dbDumpTrie(ctx *cli.Context) error { } func freezerInspect(ctx *cli.Context) error { - var ( - start, end int64 - disableSnappy bool - err error - ) - if ctx.NArg() < 3 { + if ctx.NArg() < 4 { return fmt.Errorf("required arguments: %v", ctx.Command.ArgsUsage) } - kind := ctx.Args().Get(0) - if noSnap, ok := rawdb.FreezerNoSnappy[kind]; !ok { - var options []string - for opt := range rawdb.FreezerNoSnappy { - options = append(options, opt) - } - sort.Strings(options) - return fmt.Errorf("Could read freezer-type '%v'. Available options: %v", kind, options) - } else { - disableSnappy = noSnap - } - if start, err = strconv.ParseInt(ctx.Args().Get(1), 10, 64); err != nil { - log.Info("Could read start-param", "error", err) + var ( + freezer = ctx.Args().Get(0) + table = ctx.Args().Get(1) + ) + start, err := strconv.ParseInt(ctx.Args().Get(2), 10, 64) + if err != nil { + log.Info("Could not read start-param", "err", err) return err } - if end, err = strconv.ParseInt(ctx.Args().Get(2), 10, 64); err != nil { - log.Info("Could read count param", "error", err) + end, err := strconv.ParseInt(ctx.Args().Get(3), 10, 64) + if err != nil { + log.Info("Could not read count param", "err", err) return err } stack, _ := makeConfigNode(ctx) defer stack.Close() - path := filepath.Join(stack.ResolvePath("chaindata"), "ancient") - log.Info("Opening freezer", "location", path, "name", kind) - if f, err := rawdb.NewFreezerTable(path, kind, disableSnappy, true); err != nil { + + db := utils.MakeChainDatabase(ctx, stack, true) + defer db.Close() + + ancient, err := db.AncientDatadir() + if err != nil { + log.Info("Failed to retrieve ancient root", "err", err) return err - } else { - f.DumpIndex(start, end) } - return nil + return rawdb.InspectFreezerTable(ancient, freezer, table, start, end) } func importLDBdata(ctx *cli.Context) error { @@ -718,7 +708,7 @@ func showMetaData(ctx *cli.Context) error { if val == nil { return "" } - return fmt.Sprintf("%d (0x%x)", *val, *val) + return fmt.Sprintf("%d (%#x)", *val, *val) } data := [][]string{ {"databaseVersion", pp(rawdb.ReadDatabaseVersion(db))}, @@ -728,7 +718,7 @@ func showMetaData(ctx *cli.Context) error { if b := rawdb.ReadHeadBlock(db); b != nil { data = append(data, []string{"headBlock.Hash", fmt.Sprintf("%v", b.Hash())}) data = append(data, []string{"headBlock.Root", fmt.Sprintf("%v", b.Root())}) - data = append(data, []string{"headBlock.Number", fmt.Sprintf("%d (0x%x)", b.Number(), b.Number())}) + data = append(data, []string{"headBlock.Number", fmt.Sprintf("%d (%#x)", b.Number(), b.Number())}) } if b := rawdb.ReadSkeletonSyncStatus(db); b != nil { data = append(data, []string{"SkeletonSyncStatus", string(b)}) @@ -736,7 +726,7 @@ func showMetaData(ctx *cli.Context) error { if h := rawdb.ReadHeadHeader(db); h != nil { data = append(data, []string{"headHeader.Hash", fmt.Sprintf("%v", h.Hash())}) data = append(data, []string{"headHeader.Root", fmt.Sprintf("%v", h.Root)}) - data = append(data, []string{"headHeader.Number", fmt.Sprintf("%d (0x%x)", h.Number, h.Number)}) + data = append(data, []string{"headHeader.Number", fmt.Sprintf("%d (%#x)", h.Number, h.Number)}) } data = append(data, [][]string{{"frozen", fmt.Sprintf("%d items", ancients)}, {"lastPivotNumber", pp(rawdb.ReadLastPivotNumber(db))}, diff --git a/cmd/geth/genesis_test.go b/cmd/geth/genesis_test.go index c95755f2d9193..7667a85811589 100644 --- a/cmd/geth/genesis_test.go +++ b/cmd/geth/genesis_test.go @@ -83,7 +83,7 @@ func TestCustomGenesis(t *testing.T) { // Query the custom genesis block geth := runGeth(t, "--networkid", "1337", "--syncmode=full", "--cache", "16", - "--datadir", datadir, "--maxpeers", "0", "--port", "0", + "--datadir", datadir, "--maxpeers", "0", "--port", "0", "--authrpc.port", "0", "--nodiscover", "--nat", "none", "--ipcdisable", "--exec", tt.query, "console") geth.ExpectRegexp(tt.result) diff --git a/cmd/geth/les_test.go b/cmd/geth/les_test.go index 73cc23e6674fe..d06c3b91d8a50 100644 --- a/cmd/geth/les_test.go +++ b/cmd/geth/les_test.go @@ -81,41 +81,6 @@ func (g *gethrpc) getNodeInfo() *p2p.NodeInfo { return g.nodeInfo } -func (g *gethrpc) waitSynced() { - // Check if it's synced now - var result interface{} - g.callRPC(&result, "eth_syncing") - syncing, ok := result.(bool) - if ok && !syncing { - g.geth.Logf("%v already synced", g.name) - return - } - - // Actually wait, subscribe to the event - ch := make(chan interface{}) - sub, err := g.rpc.Subscribe(context.Background(), "eth", ch, "syncing") - if err != nil { - g.geth.Fatalf("%v syncing: %v", g.name, err) - } - defer sub.Unsubscribe() - timeout := time.After(4 * time.Second) - select { - case ev := <-ch: - g.geth.Log("'syncing' event", ev) - syncing, ok := ev.(bool) - if ok && !syncing { - break - } - g.geth.Log("Other 'syncing' event", ev) - case err := <-sub.Err(): - g.geth.Fatalf("%v notification: %v", g.name, err) - break - case <-timeout: - g.geth.Fatalf("%v timeout syncing", g.name) - break - } -} - // ipcEndpoint resolves an IPC endpoint based on a configured value, taking into // account the set data folders as well as the designated platform we're currently // running on. @@ -146,7 +111,7 @@ var nextIPC = uint32(0) func startGethWithIpc(t *testing.T, name string, args ...string) *gethrpc { ipcName := fmt.Sprintf("geth-%d.ipc", atomic.AddUint32(&nextIPC, 1)) - args = append([]string{"--networkid=42", "--port=0", "--ipcpath", ipcName}, args...) + args = append([]string{"--networkid=42", "--port=0", "--authrpc.port", "0", "--ipcpath", ipcName}, args...) t.Logf("Starting %v with rpc: %v", name, args) g := &gethrpc{ @@ -179,7 +144,7 @@ func initGeth(t *testing.T) string { func startLightServer(t *testing.T) *gethrpc { datadir := initGeth(t) t.Logf("Importing keys to geth") - runGeth(t, "--datadir", datadir, "--password", "./testdata/password.txt", "account", "import", "./testdata/key.prv", "--lightkdf").WaitExit() + runGeth(t, "account", "import", "--datadir", datadir, "--password", "./testdata/password.txt", "--lightkdf", "./testdata/key.prv").WaitExit() account := "0x02f0d131f1f97aef08aec6e3291b957d9efe7105" server := startGethWithIpc(t, "lightserver", "--allow-insecure-unlock", "--datadir", datadir, "--password", "./testdata/password.txt", "--unlock", account, "--mine", "--light.serve=100", "--light.maxpeers=1", "--nodiscover", "--nat=extip:127.0.0.1", "--verbosity=4") return server diff --git a/cmd/geth/main.go b/cmd/geth/main.go index ea8a5187804f8..b9e3ed31e813d 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -44,7 +44,7 @@ import ( _ "github.com/ethereum/go-ethereum/eth/tracers/js" _ "github.com/ethereum/go-ethereum/eth/tracers/native" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) const ( @@ -58,7 +58,7 @@ var ( // The app that holds all commands and flags. app = flags.NewApp(gitCommit, gitDate, "the go-ethereum command line interface") // flags that configure the node - nodeFlags = utils.GroupFlags([]cli.Flag{ + nodeFlags = flags.Merge([]cli.Flag{ utils.IdentityFlag, utils.UnlockedAccountFlag, utils.PasswordFileFlag, @@ -69,8 +69,8 @@ var ( utils.NoUSBFlag, utils.USBFlag, utils.SmartCardDaemonPathFlag, - utils.OverrideArrowGlacierFlag, utils.OverrideTerminalTotalDifficulty, + utils.OverrideTerminalTotalDifficultyPassed, utils.EthashCacheDirFlag, utils.EthashCachesInMemoryFlag, utils.EthashCachesOnDiskFlag, @@ -117,8 +117,10 @@ var ( utils.CacheSnapshotFlag, utils.CacheNoPrefetchFlag, utils.CachePreimagesFlag, + utils.CacheLogSizeFlag, utils.FDLimitFlag, utils.ListenPortFlag, + utils.DiscoveryPortFlag, utils.MaxPeersFlag, utils.MaxPendingPeersFlag, utils.MiningEnabledFlag, @@ -151,6 +153,7 @@ var ( utils.GpoMaxGasPriceFlag, utils.GpoIgnoreGasPriceFlag, utils.MinerNotifyFullFlag, + utils.IgnoreLegacyReceiptsFlag, configFileFlag, }, utils.NetworkFlags, utils.DatabasePathFlags) @@ -207,7 +210,7 @@ func init() { app.Action = geth app.HideVersion = true // we have a command to print the version app.Copyright = "Copyright 2013-2022 The go-ethereum Authors" - app.Commands = []cli.Command{ + app.Commands = []*cli.Command{ // See chaincmd.go: initCommand, importCommand, @@ -241,13 +244,16 @@ func init() { } sort.Sort(cli.CommandsByName(app.Commands)) - app.Flags = utils.GroupFlags(nodeFlags, + app.Flags = flags.Merge( + nodeFlags, rpcFlags, consoleFlags, debug.Flags, - metricsFlags) + metricsFlags, + ) app.Before = func(ctx *cli.Context) error { + flags.MigrateGlobalFlags(ctx) return debug.Setup(ctx) } app.After = func(ctx *cli.Context) error { @@ -269,22 +275,22 @@ func main() { func prepare(ctx *cli.Context) { // If we're running a known preset, log it for convenience. switch { - case ctx.GlobalIsSet(utils.RopstenFlag.Name): + case ctx.IsSet(utils.RopstenFlag.Name): log.Info("Starting Geth on Ropsten testnet...") - case ctx.GlobalIsSet(utils.RinkebyFlag.Name): + case ctx.IsSet(utils.RinkebyFlag.Name): log.Info("Starting Geth on Rinkeby testnet...") - case ctx.GlobalIsSet(utils.GoerliFlag.Name): + case ctx.IsSet(utils.GoerliFlag.Name): log.Info("Starting Geth on Görli testnet...") - case ctx.GlobalIsSet(utils.SepoliaFlag.Name): + case ctx.IsSet(utils.SepoliaFlag.Name): log.Info("Starting Geth on Sepolia testnet...") - case ctx.GlobalIsSet(utils.KilnFlag.Name): + case ctx.IsSet(utils.KilnFlag.Name): log.Info("Starting Geth on Kiln testnet...") - case ctx.GlobalIsSet(utils.DeveloperFlag.Name): + case ctx.IsSet(utils.DeveloperFlag.Name): log.Info("Starting Geth in ephemeral dev mode...") log.Warn(`You are running Geth in --dev mode. Please note the following: @@ -302,27 +308,27 @@ func prepare(ctx *cli.Context) { to 0, and discovery is disabled. `) - case !ctx.GlobalIsSet(utils.NetworkIdFlag.Name): + case !ctx.IsSet(utils.NetworkIdFlag.Name): log.Info("Starting Geth on Ethereum mainnet...") } // If we're a full node on mainnet without --cache specified, bump default cache allowance - if ctx.GlobalString(utils.SyncModeFlag.Name) != "light" && !ctx.GlobalIsSet(utils.CacheFlag.Name) && !ctx.GlobalIsSet(utils.NetworkIdFlag.Name) { + if ctx.String(utils.SyncModeFlag.Name) != "light" && !ctx.IsSet(utils.CacheFlag.Name) && !ctx.IsSet(utils.NetworkIdFlag.Name) { // Make sure we're not on any supported preconfigured testnet either - if !ctx.GlobalIsSet(utils.RopstenFlag.Name) && - !ctx.GlobalIsSet(utils.SepoliaFlag.Name) && - !ctx.GlobalIsSet(utils.RinkebyFlag.Name) && - !ctx.GlobalIsSet(utils.GoerliFlag.Name) && - !ctx.GlobalIsSet(utils.KilnFlag.Name) && - !ctx.GlobalIsSet(utils.DeveloperFlag.Name) { + if !ctx.IsSet(utils.RopstenFlag.Name) && + !ctx.IsSet(utils.SepoliaFlag.Name) && + !ctx.IsSet(utils.RinkebyFlag.Name) && + !ctx.IsSet(utils.GoerliFlag.Name) && + !ctx.IsSet(utils.KilnFlag.Name) && + !ctx.IsSet(utils.DeveloperFlag.Name) { // Nope, we're really on mainnet. Bump that cache up! - log.Info("Bumping default cache on mainnet", "provided", ctx.GlobalInt(utils.CacheFlag.Name), "updated", 4096) - ctx.GlobalSet(utils.CacheFlag.Name, strconv.Itoa(4096)) + log.Info("Bumping default cache on mainnet", "provided", ctx.Int(utils.CacheFlag.Name), "updated", 4096) + ctx.Set(utils.CacheFlag.Name, strconv.Itoa(4096)) } } // If we're running a light client on any network, drop the cache to some meaningfully low amount - if ctx.GlobalString(utils.SyncModeFlag.Name) == "light" && !ctx.GlobalIsSet(utils.CacheFlag.Name) { - log.Info("Dropping default light client cache", "provided", ctx.GlobalInt(utils.CacheFlag.Name), "updated", 128) - ctx.GlobalSet(utils.CacheFlag.Name, strconv.Itoa(128)) + if ctx.String(utils.SyncModeFlag.Name) == "light" && !ctx.IsSet(utils.CacheFlag.Name) { + log.Info("Dropping default light client cache", "provided", ctx.Int(utils.CacheFlag.Name), "updated", 128) + ctx.Set(utils.CacheFlag.Name, strconv.Itoa(128)) } // Start metrics export if enabled @@ -336,7 +342,7 @@ func prepare(ctx *cli.Context) { // It creates a default node based on the command line arguments and runs it in // blocking mode, waiting for it to be shut down. func geth(ctx *cli.Context) error { - if args := ctx.Args(); len(args) > 0 { + if args := ctx.Args().Slice(); len(args) > 0 { return fmt.Errorf("invalid command: %q", args[0]) } @@ -407,7 +413,7 @@ func startNode(ctx *cli.Context, stack *node.Node, backend ethapi.Backend, isCon // Spawn a standalone goroutine for status synchronization monitoring, // close the node when synchronization is complete if user required. - if ctx.GlobalBool(utils.ExitWhenSyncedFlag.Name) { + if ctx.Bool(utils.ExitWhenSyncedFlag.Name) { go func() { sub := stack.EventMux().Subscribe(downloader.DoneEvent{}) defer sub.Unsubscribe() @@ -430,9 +436,9 @@ func startNode(ctx *cli.Context, stack *node.Node, backend ethapi.Backend, isCon } // Start auxiliary services if enabled - if ctx.GlobalBool(utils.MiningEnabledFlag.Name) || ctx.GlobalBool(utils.DeveloperFlag.Name) { + if ctx.Bool(utils.MiningEnabledFlag.Name) || ctx.Bool(utils.DeveloperFlag.Name) { // Mining only makes sense if a full Ethereum node is running - if ctx.GlobalString(utils.SyncModeFlag.Name) == "light" { + if ctx.String(utils.SyncModeFlag.Name) == "light" { utils.Fatalf("Light clients do not support mining") } ethBackend, ok := backend.(*eth.EthAPIBackend) @@ -440,10 +446,10 @@ func startNode(ctx *cli.Context, stack *node.Node, backend ethapi.Backend, isCon utils.Fatalf("Ethereum service not running") } // Set the gas price to the limits from the CLI and start mining - gasprice := utils.GlobalBig(ctx, utils.MinerGasPriceFlag.Name) + gasprice := flags.GlobalBig(ctx, utils.MinerGasPriceFlag.Name) ethBackend.TxPool().SetGasPrice(gasprice) // start mining - threads := ctx.GlobalInt(utils.MinerThreadsFlag.Name) + threads := ctx.Int(utils.MinerThreadsFlag.Name) if err := ethBackend.StartMining(threads); err != nil { utils.Fatalf("Failed to start mining: %v", err) } @@ -453,7 +459,7 @@ func startNode(ctx *cli.Context, stack *node.Node, backend ethapi.Backend, isCon // unlockAccounts unlocks any account specifically requested. func unlockAccounts(ctx *cli.Context, stack *node.Node) { var unlocks []string - inputs := strings.Split(ctx.GlobalString(utils.UnlockedAccountFlag.Name), ",") + inputs := strings.Split(ctx.String(utils.UnlockedAccountFlag.Name), ",") for _, input := range inputs { if trimmed := strings.TrimSpace(input); trimmed != "" { unlocks = append(unlocks, trimmed) diff --git a/cmd/geth/misccmd.go b/cmd/geth/misccmd.go index b347d31d97e18..cc5feea9fbc9f 100644 --- a/cmd/geth/misccmd.go +++ b/cmd/geth/misccmd.go @@ -26,28 +26,27 @@ import ( "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/params" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) var ( - VersionCheckUrlFlag = cli.StringFlag{ + VersionCheckUrlFlag = &cli.StringFlag{ Name: "check.url", Usage: "URL to use when checking vulnerabilities", Value: "https://geth.ethereum.org/docs/vulnerabilities/vulnerabilities.json", } - VersionCheckVersionFlag = cli.StringFlag{ + VersionCheckVersionFlag = &cli.StringFlag{ Name: "check.version", Usage: "Version to check", Value: fmt.Sprintf("Geth/v%v/%v-%v/%v", params.VersionWithCommit(gitCommit, gitDate), runtime.GOOS, runtime.GOARCH, runtime.Version()), } - makecacheCommand = cli.Command{ - Action: utils.MigrateFlags(makecache), + makecacheCommand = &cli.Command{ + Action: makecache, Name: "makecache", Usage: "Generate ethash verification cache (for testing)", ArgsUsage: " ", - Category: "MISCELLANEOUS COMMANDS", Description: ` The makecache command generates an ethash cache in . @@ -55,12 +54,11 @@ This command exists to support the system testing project. Regular users do not need to execute it. `, } - makedagCommand = cli.Command{ - Action: utils.MigrateFlags(makedag), + makedagCommand = &cli.Command{ + Action: makedag, Name: "makedag", Usage: "Generate ethash mining DAG (for testing)", ArgsUsage: " ", - Category: "MISCELLANEOUS COMMANDS", Description: ` The makedag command generates an ethash DAG in . @@ -68,43 +66,40 @@ This command exists to support the system testing project. Regular users do not need to execute it. `, } - versionCommand = cli.Command{ - Action: utils.MigrateFlags(version), + versionCommand = &cli.Command{ + Action: version, Name: "version", Usage: "Print version numbers", ArgsUsage: " ", - Category: "MISCELLANEOUS COMMANDS", Description: ` The output of this command is supposed to be machine-readable. `, } - versionCheckCommand = cli.Command{ - Action: utils.MigrateFlags(versionCheck), + versionCheckCommand = &cli.Command{ + Action: versionCheck, Flags: []cli.Flag{ VersionCheckUrlFlag, VersionCheckVersionFlag, }, Name: "version-check", - Usage: "Checks (online) whether the current version suffers from any known security vulnerabilities", + Usage: "Checks (online) for known Geth security vulnerabilities", ArgsUsage: "", - Category: "MISCELLANEOUS COMMANDS", Description: ` The version-check command fetches vulnerability-information from https://geth.ethereum.org/docs/vulnerabilities/vulnerabilities.json, and displays information about any security vulnerabilities that affect the currently executing version. `, } - licenseCommand = cli.Command{ - Action: utils.MigrateFlags(license), + licenseCommand = &cli.Command{ + Action: license, Name: "license", Usage: "Display license information", ArgsUsage: " ", - Category: "MISCELLANEOUS COMMANDS", } ) // makecache generates an ethash verification cache into the provided folder. func makecache(ctx *cli.Context) error { - args := ctx.Args() + args := ctx.Args().Slice() if len(args) != 2 { utils.Fatalf(`Usage: geth makecache `) } @@ -119,7 +114,7 @@ func makecache(ctx *cli.Context) error { // makedag generates an ethash mining DAG into the provided folder. func makedag(ctx *cli.Context) error { - args := ctx.Args() + args := ctx.Args().Slice() if len(args) != 2 { utils.Fatalf(`Usage: geth makedag `) } diff --git a/cmd/geth/snapshot.go b/cmd/geth/snapshot.go index 286eeed8ed2d5..39bef1f2d3525 100644 --- a/cmd/geth/snapshot.go +++ b/cmd/geth/snapshot.go @@ -31,10 +31,11 @@ import ( "github.com/ethereum/go-ethereum/core/state/snapshot" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/internal/flags" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" - cli "gopkg.in/urfave/cli.v1" + cli "github.com/urfave/cli/v2" ) var ( @@ -46,19 +47,17 @@ var ( ) var ( - snapshotCommand = cli.Command{ + snapshotCommand = &cli.Command{ Name: "snapshot", Usage: "A set of commands based on the snapshot", - Category: "MISCELLANEOUS COMMANDS", Description: "", - Subcommands: []cli.Command{ + Subcommands: []*cli.Command{ { Name: "prune-state", Usage: "Prune stale ethereum state data based on the snapshot", ArgsUsage: "", - Action: utils.MigrateFlags(pruneState), - Category: "MISCELLANEOUS COMMANDS", - Flags: utils.GroupFlags([]cli.Flag{ + Action: pruneState, + Flags: flags.Merge([]cli.Flag{ utils.CacheTrieJournalFlag, utils.BloomFilterSizeFlag, }, utils.NetworkFlags, utils.DatabasePathFlags), @@ -81,9 +80,8 @@ the trie clean cache with default directory will be deleted. Name: "verify-state", Usage: "Recalculate state hash based on the snapshot for verification", ArgsUsage: "", - Action: utils.MigrateFlags(verifyState), - Category: "MISCELLANEOUS COMMANDS", - Flags: utils.GroupFlags(utils.NetworkFlags, utils.DatabasePathFlags), + Action: verifyState, + Flags: flags.Merge(utils.NetworkFlags, utils.DatabasePathFlags), Description: ` geth snapshot verify-state will traverse the whole accounts and storages set based on the specified @@ -95,21 +93,30 @@ In other words, this command does the snapshot to trie conversion. Name: "check-dangling-storage", Usage: "Check that there is no 'dangling' snap storage", ArgsUsage: "", - Action: utils.MigrateFlags(checkDanglingStorage), - Category: "MISCELLANEOUS COMMANDS", - Flags: utils.GroupFlags(utils.NetworkFlags, utils.DatabasePathFlags), + Action: checkDanglingStorage, + Flags: flags.Merge(utils.NetworkFlags, utils.DatabasePathFlags), Description: ` geth snapshot check-dangling-storage traverses the snap storage data, and verifies that all snapshot storage data has a corresponding account. +`, + }, + { + Name: "inspect-account", + Usage: "Check all snapshot layers for the a specific account", + ArgsUsage: "
", + Action: checkAccount, + Flags: flags.Merge(utils.NetworkFlags, utils.DatabasePathFlags), + Description: ` +geth snapshot inspect-account
checks all snapshot layers and prints out +information about the specified address. `, }, { Name: "traverse-state", - Usage: "Traverse the state with given root hash for verification", + Usage: "Traverse the state with given root hash and perform quick verification", ArgsUsage: "", - Action: utils.MigrateFlags(traverseState), - Category: "MISCELLANEOUS COMMANDS", - Flags: utils.GroupFlags(utils.NetworkFlags, utils.DatabasePathFlags), + Action: traverseState, + Flags: flags.Merge(utils.NetworkFlags, utils.DatabasePathFlags), Description: ` geth snapshot traverse-state will traverse the whole state from the given state root and will abort if any @@ -121,11 +128,10 @@ It's also usable without snapshot enabled. }, { Name: "traverse-rawstate", - Usage: "Traverse the state with given root hash for verification", + Usage: "Traverse the state with given root hash and perform detailed verification", ArgsUsage: "", - Action: utils.MigrateFlags(traverseRawState), - Category: "MISCELLANEOUS COMMANDS", - Flags: utils.GroupFlags(utils.NetworkFlags, utils.DatabasePathFlags), + Action: traverseRawState, + Flags: flags.Merge(utils.NetworkFlags, utils.DatabasePathFlags), Description: ` geth snapshot traverse-rawstate will traverse the whole state from the given root and will abort if any referenced @@ -140,9 +146,8 @@ It's also usable without snapshot enabled. Name: "dump", Usage: "Dump a specific block from storage (same as 'geth dump' but using snapshots)", ArgsUsage: "[? | ]", - Action: utils.MigrateFlags(dumpState), - Category: "MISCELLANEOUS COMMANDS", - Flags: utils.GroupFlags([]cli.Flag{ + Action: dumpState, + Flags: flags.Merge([]cli.Flag{ utils.ExcludeCodeFlag, utils.ExcludeStorageFlag, utils.StartKeyFlag, @@ -165,7 +170,7 @@ func pruneState(ctx *cli.Context) error { defer stack.Close() chaindb := utils.MakeChainDatabase(ctx, stack, false) - pruner, err := pruner.NewPruner(chaindb, stack.ResolvePath(""), stack.ResolvePath(config.Eth.TrieCleanCacheJournal), ctx.GlobalUint64(utils.BloomFilterSizeFlag.Name)) + pruner, err := pruner.NewPruner(chaindb, stack.ResolvePath(""), stack.ResolvePath(config.Eth.TrieCleanCacheJournal), ctx.Uint64(utils.BloomFilterSizeFlag.Name)) if err != nil { log.Error("Failed to open snapshot tree", "err", err) return err @@ -176,7 +181,7 @@ func pruneState(ctx *cli.Context) error { } var targetRoot common.Hash if ctx.NArg() == 1 { - targetRoot, err = parseRoot(ctx.Args()[0]) + targetRoot, err = parseRoot(ctx.Args().First()) if err != nil { log.Error("Failed to resolve state root", "err", err) return err @@ -210,7 +215,7 @@ func verifyState(ctx *cli.Context) error { } var root = headBlock.Root() if ctx.NArg() == 1 { - root, err = parseRoot(ctx.Args()[0]) + root, err = parseRoot(ctx.Args().First()) if err != nil { log.Error("Failed to resolve state root", "err", err) return err @@ -255,7 +260,7 @@ func traverseState(ctx *cli.Context) error { err error ) if ctx.NArg() == 1 { - root, err = parseRoot(ctx.Args()[0]) + root, err = parseRoot(ctx.Args().First()) if err != nil { log.Error("Failed to resolve state root", "err", err) return err @@ -266,7 +271,7 @@ func traverseState(ctx *cli.Context) error { log.Info("Start traversing the state", "root", root, "number", headBlock.NumberU64()) } triedb := trie.NewDatabase(chaindb) - t, err := trie.NewSecure(root, triedb) + t, err := trie.NewStateTrie(common.Hash{}, root, triedb) if err != nil { log.Error("Failed to open trie", "root", root, "err", err) return err @@ -287,7 +292,7 @@ func traverseState(ctx *cli.Context) error { return err } if acc.Root != emptyRoot { - storageTrie, err := trie.NewSecure(acc.Root, triedb) + storageTrie, err := trie.NewStateTrie(common.BytesToHash(accIter.Key), acc.Root, triedb) if err != nil { log.Error("Failed to open storage trie", "root", acc.Root, "err", err) return err @@ -344,7 +349,7 @@ func traverseRawState(ctx *cli.Context) error { err error ) if ctx.NArg() == 1 { - root, err = parseRoot(ctx.Args()[0]) + root, err = parseRoot(ctx.Args().First()) if err != nil { log.Error("Failed to resolve state root", "err", err) return err @@ -355,7 +360,7 @@ func traverseRawState(ctx *cli.Context) error { log.Info("Start traversing the state", "root", root, "number", headBlock.NumberU64()) } triedb := trie.NewDatabase(chaindb) - t, err := trie.NewSecure(root, triedb) + t, err := trie.NewStateTrie(common.Hash{}, root, triedb) if err != nil { log.Error("Failed to open trie", "root", root, "err", err) return err @@ -367,6 +372,8 @@ func traverseRawState(ctx *cli.Context) error { codes int lastReport time.Time start = time.Now() + hasher = crypto.NewKeccakState() + got = make([]byte, 32) ) accIter := t.NodeIterator(nil) for accIter.Next(true) { @@ -376,10 +383,18 @@ func traverseRawState(ctx *cli.Context) error { // Check the present for non-empty hash node(embedded node doesn't // have their own hash). if node != (common.Hash{}) { - if !rawdb.HasTrieNode(chaindb, node) { + blob := rawdb.ReadTrieNode(chaindb, node) + if len(blob) == 0 { log.Error("Missing trie node(account)", "hash", node) return errors.New("missing account") } + hasher.Reset() + hasher.Write(blob) + hasher.Read(got) + if !bytes.Equal(got, node.Bytes()) { + log.Error("Invalid trie node(account)", "hash", node.Hex(), "value", blob) + return errors.New("invalid account node") + } } // If it's a leaf node, yes we are touching an account, // dig into the storage trie further. @@ -391,7 +406,7 @@ func traverseRawState(ctx *cli.Context) error { return errors.New("invalid account") } if acc.Root != emptyRoot { - storageTrie, err := trie.NewSecure(acc.Root, triedb) + storageTrie, err := trie.NewStateTrie(common.BytesToHash(accIter.LeafKey()), acc.Root, triedb) if err != nil { log.Error("Failed to open storage trie", "root", acc.Root, "err", err) return errors.New("missing storage trie") @@ -404,10 +419,18 @@ func traverseRawState(ctx *cli.Context) error { // Check the present for non-empty hash node(embedded node doesn't // have their own hash). if node != (common.Hash{}) { - if !rawdb.HasTrieNode(chaindb, node) { + blob := rawdb.ReadTrieNode(chaindb, node) + if len(blob) == 0 { log.Error("Missing trie node(storage)", "hash", node) return errors.New("missing storage") } + hasher.Reset() + hasher.Write(blob) + hasher.Read(got) + if !bytes.Equal(got, node.Bytes()) { + log.Error("Invalid trie node(storage)", "hash", node.Hex(), "value", blob) + return errors.New("invalid storage node") + } } // Bump the counter if it's leaf node. if storageIter.Leaf() { @@ -517,3 +540,35 @@ func dumpState(ctx *cli.Context) error { "elapsed", common.PrettyDuration(time.Since(start))) return nil } + +// checkAccount iterates the snap data layers, and looks up the given account +// across all layers. +func checkAccount(ctx *cli.Context) error { + if ctx.NArg() != 1 { + return errors.New("need arg") + } + var ( + hash common.Hash + addr common.Address + ) + switch arg := ctx.Args().First(); len(arg) { + case 40, 42: + addr = common.HexToAddress(arg) + hash = crypto.Keccak256Hash(addr.Bytes()) + case 64, 66: + hash = common.HexToHash(arg) + default: + return errors.New("malformed address or hash") + } + stack, _ := makeConfigNode(ctx) + defer stack.Close() + chaindb := utils.MakeChainDatabase(ctx, stack, true) + defer chaindb.Close() + start := time.Now() + log.Info("Checking difflayer journal", "address", addr, "hash", hash) + if err := snapshot.CheckJournalAccount(chaindb, hash); err != nil { + return err + } + log.Info("Checked the snapshot journalled storage", "time", common.PrettyDuration(time.Since(start))) + return nil +} diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go deleted file mode 100644 index 731992ff7c219..0000000000000 --- a/cmd/geth/usage.go +++ /dev/null @@ -1,300 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of go-ethereum. -// -// go-ethereum is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// go-ethereum is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with go-ethereum. If not, see . - -// Contains the geth command usage template and generator. - -package main - -import ( - "io" - "sort" - - "github.com/ethereum/go-ethereum/cmd/utils" - "github.com/ethereum/go-ethereum/internal/debug" - "github.com/ethereum/go-ethereum/internal/flags" - "gopkg.in/urfave/cli.v1" -) - -// AppHelpFlagGroups is the application flags, grouped by functionality. -var AppHelpFlagGroups = []flags.FlagGroup{ - { - Name: "ETHEREUM", - Flags: utils.GroupFlags([]cli.Flag{ - configFileFlag, - utils.MinFreeDiskSpaceFlag, - utils.KeyStoreDirFlag, - utils.USBFlag, - utils.SmartCardDaemonPathFlag, - utils.NetworkIdFlag, - utils.SyncModeFlag, - utils.ExitWhenSyncedFlag, - utils.GCModeFlag, - utils.TxLookupLimitFlag, - utils.EthStatsURLFlag, - utils.IdentityFlag, - utils.LightKDFFlag, - utils.EthRequiredBlocksFlag, - }, utils.NetworkFlags, utils.DatabasePathFlags), - }, - { - Name: "LIGHT CLIENT", - Flags: []cli.Flag{ - utils.LightServeFlag, - utils.LightIngressFlag, - utils.LightEgressFlag, - utils.LightMaxPeersFlag, - utils.UltraLightServersFlag, - utils.UltraLightFractionFlag, - utils.UltraLightOnlyAnnounceFlag, - utils.LightNoPruneFlag, - utils.LightNoSyncServeFlag, - }, - }, - { - Name: "DEVELOPER CHAIN", - Flags: []cli.Flag{ - utils.DeveloperFlag, - utils.DeveloperPeriodFlag, - utils.DeveloperGasLimitFlag, - }, - }, - { - Name: "ETHASH", - Flags: []cli.Flag{ - utils.EthashCacheDirFlag, - utils.EthashCachesInMemoryFlag, - utils.EthashCachesOnDiskFlag, - utils.EthashCachesLockMmapFlag, - utils.EthashDatasetDirFlag, - utils.EthashDatasetsInMemoryFlag, - utils.EthashDatasetsOnDiskFlag, - utils.EthashDatasetsLockMmapFlag, - }, - }, - { - Name: "TRANSACTION POOL", - Flags: []cli.Flag{ - utils.TxPoolLocalsFlag, - utils.TxPoolNoLocalsFlag, - utils.TxPoolJournalFlag, - utils.TxPoolRejournalFlag, - utils.TxPoolPriceLimitFlag, - utils.TxPoolPriceBumpFlag, - utils.TxPoolAccountSlotsFlag, - utils.TxPoolGlobalSlotsFlag, - utils.TxPoolAccountQueueFlag, - utils.TxPoolGlobalQueueFlag, - utils.TxPoolLifetimeFlag, - }, - }, - { - Name: "PERFORMANCE TUNING", - Flags: []cli.Flag{ - utils.CacheFlag, - utils.CacheDatabaseFlag, - utils.CacheTrieFlag, - utils.CacheTrieJournalFlag, - utils.CacheTrieRejournalFlag, - utils.CacheGCFlag, - utils.CacheSnapshotFlag, - utils.CacheNoPrefetchFlag, - utils.CachePreimagesFlag, - utils.FDLimitFlag, - }, - }, - { - Name: "ACCOUNT", - Flags: []cli.Flag{ - utils.UnlockedAccountFlag, - utils.PasswordFileFlag, - utils.ExternalSignerFlag, - utils.InsecureUnlockAllowedFlag, - }, - }, - { - Name: "API AND CONSOLE", - Flags: []cli.Flag{ - utils.IPCDisabledFlag, - utils.IPCPathFlag, - utils.HTTPEnabledFlag, - utils.HTTPListenAddrFlag, - utils.HTTPPortFlag, - utils.HTTPApiFlag, - utils.HTTPPathPrefixFlag, - utils.HTTPCORSDomainFlag, - utils.HTTPVirtualHostsFlag, - utils.WSEnabledFlag, - utils.WSListenAddrFlag, - utils.WSPortFlag, - utils.WSApiFlag, - utils.WSPathPrefixFlag, - utils.WSAllowedOriginsFlag, - utils.JWTSecretFlag, - utils.AuthListenFlag, - utils.AuthPortFlag, - utils.AuthVirtualHostsFlag, - utils.GraphQLEnabledFlag, - utils.GraphQLCORSDomainFlag, - utils.GraphQLVirtualHostsFlag, - utils.RPCGlobalGasCapFlag, - utils.RPCGlobalEVMTimeoutFlag, - utils.RPCGlobalTxFeeCapFlag, - utils.AllowUnprotectedTxs, - utils.JSpathFlag, - utils.ExecFlag, - utils.PreloadJSFlag, - }, - }, - { - Name: "NETWORKING", - Flags: []cli.Flag{ - utils.BootnodesFlag, - utils.DNSDiscoveryFlag, - utils.ListenPortFlag, - utils.MaxPeersFlag, - utils.MaxPendingPeersFlag, - utils.NATFlag, - utils.NoDiscoverFlag, - utils.DiscoveryV5Flag, - utils.NetrestrictFlag, - utils.NodeKeyFileFlag, - utils.NodeKeyHexFlag, - }, - }, - { - Name: "MINER", - Flags: []cli.Flag{ - utils.MiningEnabledFlag, - utils.MinerThreadsFlag, - utils.MinerNotifyFlag, - utils.MinerNotifyFullFlag, - utils.MinerGasPriceFlag, - utils.MinerGasLimitFlag, - utils.MinerEtherbaseFlag, - utils.MinerExtraDataFlag, - utils.MinerRecommitIntervalFlag, - utils.MinerNoVerifyFlag, - }, - }, - { - Name: "GAS PRICE ORACLE", - Flags: []cli.Flag{ - utils.GpoBlocksFlag, - utils.GpoPercentileFlag, - utils.GpoMaxGasPriceFlag, - utils.GpoIgnoreGasPriceFlag, - }, - }, - { - Name: "VIRTUAL MACHINE", - Flags: []cli.Flag{ - utils.VMEnableDebugFlag, - }, - }, - { - Name: "LOGGING AND DEBUGGING", - Flags: append([]cli.Flag{ - utils.FakePoWFlag, - utils.NoCompactionFlag, - }, debug.Flags...), - }, - { - Name: "METRICS AND STATS", - Flags: metricsFlags, - }, - { - Name: "ALIASED (deprecated)", - Flags: []cli.Flag{ - utils.NoUSBFlag, - utils.LegacyWhitelistFlag, - }, - }, - { - Name: "MISC", - Flags: []cli.Flag{ - utils.SnapshotFlag, - utils.BloomFilterSizeFlag, - cli.HelpFlag, - }, - }, -} - -func init() { - // Override the default app help template - cli.AppHelpTemplate = flags.AppHelpTemplate - - // Override the default app help printer, but only for the global app help - originalHelpPrinter := cli.HelpPrinter - cli.HelpPrinter = func(w io.Writer, tmpl string, data interface{}) { - if tmpl == flags.AppHelpTemplate { - // Iterate over all the flags and add any uncategorized ones - categorized := make(map[string]struct{}) - for _, group := range AppHelpFlagGroups { - for _, flag := range group.Flags { - categorized[flag.String()] = struct{}{} - } - } - deprecated := make(map[string]struct{}) - for _, flag := range utils.DeprecatedFlags { - deprecated[flag.String()] = struct{}{} - } - // Only add uncategorized flags if they are not deprecated - var uncategorized []cli.Flag - for _, flag := range data.(*cli.App).Flags { - if _, ok := categorized[flag.String()]; !ok { - if _, ok := deprecated[flag.String()]; !ok { - uncategorized = append(uncategorized, flag) - } - } - } - if len(uncategorized) > 0 { - // Append all ungategorized options to the misc group - miscs := len(AppHelpFlagGroups[len(AppHelpFlagGroups)-1].Flags) - AppHelpFlagGroups[len(AppHelpFlagGroups)-1].Flags = append(AppHelpFlagGroups[len(AppHelpFlagGroups)-1].Flags, uncategorized...) - - // Make sure they are removed afterwards - defer func() { - AppHelpFlagGroups[len(AppHelpFlagGroups)-1].Flags = AppHelpFlagGroups[len(AppHelpFlagGroups)-1].Flags[:miscs] - }() - } - // Render out custom usage screen - originalHelpPrinter(w, tmpl, flags.HelpData{App: data, FlagGroups: AppHelpFlagGroups}) - } else if tmpl == flags.CommandHelpTemplate { - // Iterate over all command specific flags and categorize them - categorized := make(map[string][]cli.Flag) - for _, flag := range data.(cli.Command).Flags { - if _, ok := categorized[flag.String()]; !ok { - categorized[flags.FlagCategory(flag, AppHelpFlagGroups)] = append(categorized[flags.FlagCategory(flag, AppHelpFlagGroups)], flag) - } - } - - // sort to get a stable ordering - sorted := make([]flags.FlagGroup, 0, len(categorized)) - for cat, flgs := range categorized { - sorted = append(sorted, flags.FlagGroup{Name: cat, Flags: flgs}) - } - sort.Sort(flags.ByCategory(sorted)) - - // add sorted array to data and render with default printer - originalHelpPrinter(w, tmpl, map[string]interface{}{ - "cmd": data, - "categorizedFlags": sorted, - }) - } else { - originalHelpPrinter(w, tmpl, data) - } - } -} diff --git a/cmd/geth/version_check.go b/cmd/geth/version_check.go index 6eaedf373437e..237556788eb9f 100644 --- a/cmd/geth/version_check.go +++ b/cmd/geth/version_check.go @@ -28,7 +28,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/jedisct1/go-minisign" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) var gethPubKeys []string = []string{ diff --git a/cmd/geth/version_check_test.go b/cmd/geth/version_check_test.go index b841ace5b2f9a..bd4d820a79013 100644 --- a/cmd/geth/version_check_test.go +++ b/cmd/geth/version_check_test.go @@ -118,7 +118,6 @@ func TestMatching(t *testing.T) { version, vuln.Introduced, vuln.Fixed, vuln.Name, vulnIntro, current, vulnFixed) } } - } } for major := 1; major < 2; major++ { diff --git a/cmd/p2psim/main.go b/cmd/p2psim/main.go index eaa457200a431..4edb0a9ab43d6 100644 --- a/cmd/p2psim/main.go +++ b/cmd/p2psim/main.go @@ -46,71 +46,77 @@ import ( "text/tabwriter" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/internal/flags" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/p2p/simulations" "github.com/ethereum/go-ethereum/p2p/simulations/adapters" "github.com/ethereum/go-ethereum/rpc" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) var client *simulations.Client var ( // global command flags - apiFlag = cli.StringFlag{ - Name: "api", - Value: "http://localhost:8888", - Usage: "simulation API URL", - EnvVar: "P2PSIM_API_URL", + apiFlag = &cli.StringFlag{ + Name: "api", + Value: "http://localhost:8888", + Usage: "simulation API URL", + EnvVars: []string{"P2PSIM_API_URL"}, } // events subcommand flags - currentFlag = cli.BoolFlag{ + currentFlag = &cli.BoolFlag{ Name: "current", Usage: "get existing nodes and conns first", } - filterFlag = cli.StringFlag{ + filterFlag = &cli.StringFlag{ Name: "filter", Value: "", Usage: "message filter", } // node create subcommand flags - nameFlag = cli.StringFlag{ + nameFlag = &cli.StringFlag{ Name: "name", Value: "", Usage: "node name", } - servicesFlag = cli.StringFlag{ + servicesFlag = &cli.StringFlag{ Name: "services", Value: "", Usage: "node services (comma separated)", } - keyFlag = cli.StringFlag{ + keyFlag = &cli.StringFlag{ Name: "key", Value: "", Usage: "node private key (hex encoded)", } // node rpc subcommand flags - subscribeFlag = cli.BoolFlag{ + subscribeFlag = &cli.BoolFlag{ Name: "subscribe", Usage: "method is a subscription", } ) +var ( + // Git information set by linker when building with ci.go. + gitCommit string + gitDate string +) + func main() { - app := cli.NewApp() - app.Usage = "devp2p simulation command-line client" + app := flags.NewApp(gitCommit, gitDate, "devp2p simulation command-line client") app.Flags = []cli.Flag{ apiFlag, } app.Before = func(ctx *cli.Context) error { - client = simulations.NewClient(ctx.GlobalString(apiFlag.Name)) + client = simulations.NewClient(ctx.String(apiFlag.Name)) return nil } - app.Commands = []cli.Command{ + app.Commands = []*cli.Command{ { Name: "show", Usage: "show network information", @@ -139,7 +145,7 @@ func main() { Name: "node", Usage: "manage simulation nodes", Action: listNodes, - Subcommands: []cli.Command{ + Subcommands: []*cli.Command{ { Name: "list", Usage: "list nodes", @@ -204,7 +210,7 @@ func main() { } func showNetwork(ctx *cli.Context) error { - if len(ctx.Args()) != 0 { + if ctx.NArg() != 0 { return cli.ShowCommandHelp(ctx, ctx.Command.Name) } network, err := client.GetNetwork() @@ -219,7 +225,7 @@ func showNetwork(ctx *cli.Context) error { } func streamNetwork(ctx *cli.Context) error { - if len(ctx.Args()) != 0 { + if ctx.NArg() != 0 { return cli.ShowCommandHelp(ctx, ctx.Command.Name) } events := make(chan *simulations.Event) @@ -245,7 +251,7 @@ func streamNetwork(ctx *cli.Context) error { } func createSnapshot(ctx *cli.Context) error { - if len(ctx.Args()) != 0 { + if ctx.NArg() != 0 { return cli.ShowCommandHelp(ctx, ctx.Command.Name) } snap, err := client.CreateSnapshot() @@ -256,7 +262,7 @@ func createSnapshot(ctx *cli.Context) error { } func loadSnapshot(ctx *cli.Context) error { - if len(ctx.Args()) != 0 { + if ctx.NArg() != 0 { return cli.ShowCommandHelp(ctx, ctx.Command.Name) } snap := &simulations.Snapshot{} @@ -267,7 +273,7 @@ func loadSnapshot(ctx *cli.Context) error { } func listNodes(ctx *cli.Context) error { - if len(ctx.Args()) != 0 { + if ctx.NArg() != 0 { return cli.ShowCommandHelp(ctx, ctx.Command.Name) } nodes, err := client.GetNodes() @@ -292,7 +298,7 @@ func protocolList(node *p2p.NodeInfo) []string { } func createNode(ctx *cli.Context) error { - if len(ctx.Args()) != 0 { + if ctx.NArg() != 0 { return cli.ShowCommandHelp(ctx, ctx.Command.Name) } config := adapters.RandomNodeConfig() @@ -317,11 +323,10 @@ func createNode(ctx *cli.Context) error { } func showNode(ctx *cli.Context) error { - args := ctx.Args() - if len(args) != 1 { + if ctx.NArg() != 1 { return cli.ShowCommandHelp(ctx, ctx.Command.Name) } - nodeName := args[0] + nodeName := ctx.Args().First() node, err := client.GetNode(nodeName) if err != nil { return err @@ -342,11 +347,10 @@ func showNode(ctx *cli.Context) error { } func startNode(ctx *cli.Context) error { - args := ctx.Args() - if len(args) != 1 { + if ctx.NArg() != 1 { return cli.ShowCommandHelp(ctx, ctx.Command.Name) } - nodeName := args[0] + nodeName := ctx.Args().First() if err := client.StartNode(nodeName); err != nil { return err } @@ -355,11 +359,10 @@ func startNode(ctx *cli.Context) error { } func stopNode(ctx *cli.Context) error { - args := ctx.Args() - if len(args) != 1 { + if ctx.NArg() != 1 { return cli.ShowCommandHelp(ctx, ctx.Command.Name) } - nodeName := args[0] + nodeName := ctx.Args().First() if err := client.StopNode(nodeName); err != nil { return err } @@ -368,12 +371,12 @@ func stopNode(ctx *cli.Context) error { } func connectNode(ctx *cli.Context) error { - args := ctx.Args() - if len(args) != 2 { + if ctx.NArg() != 2 { return cli.ShowCommandHelp(ctx, ctx.Command.Name) } - nodeName := args[0] - peerName := args[1] + args := ctx.Args() + nodeName := args.Get(0) + peerName := args.Get(1) if err := client.ConnectNode(nodeName, peerName); err != nil { return err } @@ -383,11 +386,11 @@ func connectNode(ctx *cli.Context) error { func disconnectNode(ctx *cli.Context) error { args := ctx.Args() - if len(args) != 2 { + if args.Len() != 2 { return cli.ShowCommandHelp(ctx, ctx.Command.Name) } - nodeName := args[0] - peerName := args[1] + nodeName := args.Get(0) + peerName := args.Get(1) if err := client.DisconnectNode(nodeName, peerName); err != nil { return err } @@ -397,21 +400,21 @@ func disconnectNode(ctx *cli.Context) error { func rpcNode(ctx *cli.Context) error { args := ctx.Args() - if len(args) < 2 { + if args.Len() < 2 { return cli.ShowCommandHelp(ctx, ctx.Command.Name) } - nodeName := args[0] - method := args[1] + nodeName := args.Get(0) + method := args.Get(1) rpcClient, err := client.RPCClient(context.Background(), nodeName) if err != nil { return err } if ctx.Bool(subscribeFlag.Name) { - return rpcSubscribe(rpcClient, ctx.App.Writer, method, args[3:]...) + return rpcSubscribe(rpcClient, ctx.App.Writer, method, args.Slice()[3:]...) } var result interface{} - params := make([]interface{}, len(args[3:])) - for i, v := range args[3:] { + params := make([]interface{}, len(args.Slice()[3:])) + for i, v := range args.Slice()[3:] { params[i] = v } if err := rpcClient.Call(&result, method, params...); err != nil { diff --git a/cmd/puppeth/genesis.go b/cmd/puppeth/genesis.go deleted file mode 100644 index ef1f977bf09f3..0000000000000 --- a/cmd/puppeth/genesis.go +++ /dev/null @@ -1,626 +0,0 @@ -// Copyright 2017 The go-ethereum Authors -// This file is part of go-ethereum. -// -// go-ethereum is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// go-ethereum is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with go-ethereum. If not, see . - -package main - -import ( - "errors" - "math" - "math/big" - "strings" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/hexutil" - math2 "github.com/ethereum/go-ethereum/common/math" - "github.com/ethereum/go-ethereum/consensus/ethash" - "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/params" -) - -// alethGenesisSpec represents the genesis specification format used by the -// C++ Ethereum implementation. -type alethGenesisSpec struct { - SealEngine string `json:"sealEngine"` - Params struct { - AccountStartNonce math2.HexOrDecimal64 `json:"accountStartNonce"` - MaximumExtraDataSize hexutil.Uint64 `json:"maximumExtraDataSize"` - HomesteadForkBlock *hexutil.Big `json:"homesteadForkBlock,omitempty"` - DaoHardforkBlock math2.HexOrDecimal64 `json:"daoHardforkBlock"` - EIP150ForkBlock *hexutil.Big `json:"EIP150ForkBlock,omitempty"` - EIP158ForkBlock *hexutil.Big `json:"EIP158ForkBlock,omitempty"` - ByzantiumForkBlock *hexutil.Big `json:"byzantiumForkBlock,omitempty"` - ConstantinopleForkBlock *hexutil.Big `json:"constantinopleForkBlock,omitempty"` - ConstantinopleFixForkBlock *hexutil.Big `json:"constantinopleFixForkBlock,omitempty"` - IstanbulForkBlock *hexutil.Big `json:"istanbulForkBlock,omitempty"` - MinGasLimit hexutil.Uint64 `json:"minGasLimit"` - MaxGasLimit hexutil.Uint64 `json:"maxGasLimit"` - TieBreakingGas bool `json:"tieBreakingGas"` - GasLimitBoundDivisor math2.HexOrDecimal64 `json:"gasLimitBoundDivisor"` - MinimumDifficulty *hexutil.Big `json:"minimumDifficulty"` - DifficultyBoundDivisor *math2.HexOrDecimal256 `json:"difficultyBoundDivisor"` - DurationLimit *math2.HexOrDecimal256 `json:"durationLimit"` - BlockReward *hexutil.Big `json:"blockReward"` - NetworkID hexutil.Uint64 `json:"networkID"` - ChainID hexutil.Uint64 `json:"chainID"` - AllowFutureBlocks bool `json:"allowFutureBlocks"` - } `json:"params"` - - Genesis struct { - Nonce types.BlockNonce `json:"nonce"` - Difficulty *hexutil.Big `json:"difficulty"` - MixHash common.Hash `json:"mixHash"` - Author common.Address `json:"author"` - Timestamp hexutil.Uint64 `json:"timestamp"` - ParentHash common.Hash `json:"parentHash"` - ExtraData hexutil.Bytes `json:"extraData"` - GasLimit hexutil.Uint64 `json:"gasLimit"` - } `json:"genesis"` - - Accounts map[common.UnprefixedAddress]*alethGenesisSpecAccount `json:"accounts"` -} - -// alethGenesisSpecAccount is the prefunded genesis account and/or precompiled -// contract definition. -type alethGenesisSpecAccount struct { - Balance *math2.HexOrDecimal256 `json:"balance,omitempty"` - Nonce uint64 `json:"nonce,omitempty"` - Precompiled *alethGenesisSpecBuiltin `json:"precompiled,omitempty"` -} - -// alethGenesisSpecBuiltin is the precompiled contract definition. -type alethGenesisSpecBuiltin struct { - Name string `json:"name,omitempty"` - StartingBlock *hexutil.Big `json:"startingBlock,omitempty"` - Linear *alethGenesisSpecLinearPricing `json:"linear,omitempty"` -} - -type alethGenesisSpecLinearPricing struct { - Base uint64 `json:"base"` - Word uint64 `json:"word"` -} - -// newAlethGenesisSpec converts a go-ethereum genesis block into a Aleth-specific -// chain specification format. -func newAlethGenesisSpec(network string, genesis *core.Genesis) (*alethGenesisSpec, error) { - // Only ethash is currently supported between go-ethereum and aleth - if genesis.Config.Ethash == nil { - return nil, errors.New("unsupported consensus engine") - } - // Reconstruct the chain spec in Aleth format - spec := &alethGenesisSpec{ - SealEngine: "Ethash", - } - // Some defaults - spec.Params.AccountStartNonce = 0 - spec.Params.TieBreakingGas = false - spec.Params.AllowFutureBlocks = false - - // Dao hardfork block is a special one. The fork block is listed as 0 in the - // config but aleth will sync with ETC clients up until the actual dao hard - // fork block. - spec.Params.DaoHardforkBlock = 0 - - if num := genesis.Config.HomesteadBlock; num != nil { - spec.Params.HomesteadForkBlock = (*hexutil.Big)(num) - } - if num := genesis.Config.EIP150Block; num != nil { - spec.Params.EIP150ForkBlock = (*hexutil.Big)(num) - } - if num := genesis.Config.EIP158Block; num != nil { - spec.Params.EIP158ForkBlock = (*hexutil.Big)(num) - } - if num := genesis.Config.ByzantiumBlock; num != nil { - spec.Params.ByzantiumForkBlock = (*hexutil.Big)(num) - } - if num := genesis.Config.ConstantinopleBlock; num != nil { - spec.Params.ConstantinopleForkBlock = (*hexutil.Big)(num) - } - if num := genesis.Config.PetersburgBlock; num != nil { - spec.Params.ConstantinopleFixForkBlock = (*hexutil.Big)(num) - } - if num := genesis.Config.IstanbulBlock; num != nil { - spec.Params.IstanbulForkBlock = (*hexutil.Big)(num) - } - spec.Params.NetworkID = (hexutil.Uint64)(genesis.Config.ChainID.Uint64()) - spec.Params.ChainID = (hexutil.Uint64)(genesis.Config.ChainID.Uint64()) - spec.Params.MaximumExtraDataSize = (hexutil.Uint64)(params.MaximumExtraDataSize) - spec.Params.MinGasLimit = (hexutil.Uint64)(params.MinGasLimit) - spec.Params.MaxGasLimit = (hexutil.Uint64)(math.MaxInt64) - spec.Params.MinimumDifficulty = (*hexutil.Big)(params.MinimumDifficulty) - spec.Params.DifficultyBoundDivisor = (*math2.HexOrDecimal256)(params.DifficultyBoundDivisor) - spec.Params.GasLimitBoundDivisor = (math2.HexOrDecimal64)(params.GasLimitBoundDivisor) - spec.Params.DurationLimit = (*math2.HexOrDecimal256)(params.DurationLimit) - spec.Params.BlockReward = (*hexutil.Big)(ethash.FrontierBlockReward) - - spec.Genesis.Nonce = types.EncodeNonce(genesis.Nonce) - spec.Genesis.MixHash = genesis.Mixhash - spec.Genesis.Difficulty = (*hexutil.Big)(genesis.Difficulty) - spec.Genesis.Author = genesis.Coinbase - spec.Genesis.Timestamp = (hexutil.Uint64)(genesis.Timestamp) - spec.Genesis.ParentHash = genesis.ParentHash - spec.Genesis.ExtraData = genesis.ExtraData - spec.Genesis.GasLimit = (hexutil.Uint64)(genesis.GasLimit) - - for address, account := range genesis.Alloc { - spec.setAccount(address, account) - } - - spec.setPrecompile(1, &alethGenesisSpecBuiltin{Name: "ecrecover", - Linear: &alethGenesisSpecLinearPricing{Base: 3000}}) - spec.setPrecompile(2, &alethGenesisSpecBuiltin{Name: "sha256", - Linear: &alethGenesisSpecLinearPricing{Base: 60, Word: 12}}) - spec.setPrecompile(3, &alethGenesisSpecBuiltin{Name: "ripemd160", - Linear: &alethGenesisSpecLinearPricing{Base: 600, Word: 120}}) - spec.setPrecompile(4, &alethGenesisSpecBuiltin{Name: "identity", - Linear: &alethGenesisSpecLinearPricing{Base: 15, Word: 3}}) - if genesis.Config.ByzantiumBlock != nil { - spec.setPrecompile(5, &alethGenesisSpecBuiltin{Name: "modexp", - StartingBlock: (*hexutil.Big)(genesis.Config.ByzantiumBlock)}) - spec.setPrecompile(6, &alethGenesisSpecBuiltin{Name: "alt_bn128_G1_add", - StartingBlock: (*hexutil.Big)(genesis.Config.ByzantiumBlock), - Linear: &alethGenesisSpecLinearPricing{Base: 500}}) - spec.setPrecompile(7, &alethGenesisSpecBuiltin{Name: "alt_bn128_G1_mul", - StartingBlock: (*hexutil.Big)(genesis.Config.ByzantiumBlock), - Linear: &alethGenesisSpecLinearPricing{Base: 40000}}) - spec.setPrecompile(8, &alethGenesisSpecBuiltin{Name: "alt_bn128_pairing_product", - StartingBlock: (*hexutil.Big)(genesis.Config.ByzantiumBlock)}) - } - if genesis.Config.IstanbulBlock != nil { - if genesis.Config.ByzantiumBlock == nil { - return nil, errors.New("invalid genesis, istanbul fork is enabled while byzantium is not") - } - spec.setPrecompile(6, &alethGenesisSpecBuiltin{ - Name: "alt_bn128_G1_add", - StartingBlock: (*hexutil.Big)(genesis.Config.ByzantiumBlock), - }) // Aleth hardcoded the gas policy - spec.setPrecompile(7, &alethGenesisSpecBuiltin{ - Name: "alt_bn128_G1_mul", - StartingBlock: (*hexutil.Big)(genesis.Config.ByzantiumBlock), - }) // Aleth hardcoded the gas policy - spec.setPrecompile(9, &alethGenesisSpecBuiltin{ - Name: "blake2_compression", - StartingBlock: (*hexutil.Big)(genesis.Config.IstanbulBlock), - }) - } - return spec, nil -} - -func (spec *alethGenesisSpec) setPrecompile(address byte, data *alethGenesisSpecBuiltin) { - if spec.Accounts == nil { - spec.Accounts = make(map[common.UnprefixedAddress]*alethGenesisSpecAccount) - } - addr := common.UnprefixedAddress(common.BytesToAddress([]byte{address})) - if _, exist := spec.Accounts[addr]; !exist { - spec.Accounts[addr] = &alethGenesisSpecAccount{} - } - spec.Accounts[addr].Precompiled = data -} - -func (spec *alethGenesisSpec) setAccount(address common.Address, account core.GenesisAccount) { - if spec.Accounts == nil { - spec.Accounts = make(map[common.UnprefixedAddress]*alethGenesisSpecAccount) - } - - a, exist := spec.Accounts[common.UnprefixedAddress(address)] - if !exist { - a = &alethGenesisSpecAccount{} - spec.Accounts[common.UnprefixedAddress(address)] = a - } - a.Balance = (*math2.HexOrDecimal256)(account.Balance) - a.Nonce = account.Nonce - -} - -// parityChainSpec is the chain specification format used by Parity. -type parityChainSpec struct { - Name string `json:"name"` - Datadir string `json:"dataDir"` - Engine struct { - Ethash struct { - Params struct { - MinimumDifficulty *hexutil.Big `json:"minimumDifficulty"` - DifficultyBoundDivisor *hexutil.Big `json:"difficultyBoundDivisor"` - DurationLimit *hexutil.Big `json:"durationLimit"` - BlockReward map[string]string `json:"blockReward"` - DifficultyBombDelays map[string]string `json:"difficultyBombDelays"` - HomesteadTransition hexutil.Uint64 `json:"homesteadTransition"` - EIP100bTransition hexutil.Uint64 `json:"eip100bTransition"` - } `json:"params"` - } `json:"Ethash"` - } `json:"engine"` - - Params struct { - AccountStartNonce hexutil.Uint64 `json:"accountStartNonce"` - MaximumExtraDataSize hexutil.Uint64 `json:"maximumExtraDataSize"` - MinGasLimit hexutil.Uint64 `json:"minGasLimit"` - GasLimitBoundDivisor math2.HexOrDecimal64 `json:"gasLimitBoundDivisor"` - NetworkID hexutil.Uint64 `json:"networkID"` - ChainID hexutil.Uint64 `json:"chainID"` - MaxCodeSize hexutil.Uint64 `json:"maxCodeSize"` - MaxCodeSizeTransition hexutil.Uint64 `json:"maxCodeSizeTransition"` - EIP98Transition hexutil.Uint64 `json:"eip98Transition"` - EIP150Transition hexutil.Uint64 `json:"eip150Transition"` - EIP160Transition hexutil.Uint64 `json:"eip160Transition"` - EIP161abcTransition hexutil.Uint64 `json:"eip161abcTransition"` - EIP161dTransition hexutil.Uint64 `json:"eip161dTransition"` - EIP155Transition hexutil.Uint64 `json:"eip155Transition"` - EIP140Transition hexutil.Uint64 `json:"eip140Transition"` - EIP211Transition hexutil.Uint64 `json:"eip211Transition"` - EIP214Transition hexutil.Uint64 `json:"eip214Transition"` - EIP658Transition hexutil.Uint64 `json:"eip658Transition"` - EIP145Transition hexutil.Uint64 `json:"eip145Transition"` - EIP1014Transition hexutil.Uint64 `json:"eip1014Transition"` - EIP1052Transition hexutil.Uint64 `json:"eip1052Transition"` - EIP1283Transition hexutil.Uint64 `json:"eip1283Transition"` - EIP1283DisableTransition hexutil.Uint64 `json:"eip1283DisableTransition"` - EIP1283ReenableTransition hexutil.Uint64 `json:"eip1283ReenableTransition"` - EIP1344Transition hexutil.Uint64 `json:"eip1344Transition"` - EIP1884Transition hexutil.Uint64 `json:"eip1884Transition"` - EIP2028Transition hexutil.Uint64 `json:"eip2028Transition"` - } `json:"params"` - - Genesis struct { - Seal struct { - Ethereum struct { - Nonce types.BlockNonce `json:"nonce"` - MixHash hexutil.Bytes `json:"mixHash"` - } `json:"ethereum"` - } `json:"seal"` - - Difficulty *hexutil.Big `json:"difficulty"` - Author common.Address `json:"author"` - Timestamp hexutil.Uint64 `json:"timestamp"` - ParentHash common.Hash `json:"parentHash"` - ExtraData hexutil.Bytes `json:"extraData"` - GasLimit hexutil.Uint64 `json:"gasLimit"` - } `json:"genesis"` - - Nodes []string `json:"nodes"` - Accounts map[common.UnprefixedAddress]*parityChainSpecAccount `json:"accounts"` -} - -// parityChainSpecAccount is the prefunded genesis account and/or precompiled -// contract definition. -type parityChainSpecAccount struct { - Balance math2.HexOrDecimal256 `json:"balance"` - Nonce math2.HexOrDecimal64 `json:"nonce,omitempty"` - Builtin *parityChainSpecBuiltin `json:"builtin,omitempty"` -} - -// parityChainSpecBuiltin is the precompiled contract definition. -type parityChainSpecBuiltin struct { - Name string `json:"name"` // Each builtin should has it own name - Pricing interface{} `json:"pricing"` // Each builtin should has it own price strategy - ActivateAt *hexutil.Big `json:"activate_at,omitempty"` // ActivateAt can't be omitted if empty, default means no fork -} - -// parityChainSpecPricing represents the different pricing models that builtin -// contracts might advertise using. -type parityChainSpecPricing struct { - Linear *parityChainSpecLinearPricing `json:"linear,omitempty"` - ModExp *parityChainSpecModExpPricing `json:"modexp,omitempty"` - - // Before the https://github.com/paritytech/parity-ethereum/pull/11039, - // Parity uses this format to config bn pairing price policy. - AltBnPairing *parityChainSepcAltBnPairingPricing `json:"alt_bn128_pairing,omitempty"` - - // Blake2F is the price per round of Blake2 compression - Blake2F *parityChainSpecBlakePricing `json:"blake2_f,omitempty"` -} - -type parityChainSpecLinearPricing struct { - Base uint64 `json:"base"` - Word uint64 `json:"word"` -} - -type parityChainSpecModExpPricing struct { - Divisor uint64 `json:"divisor"` -} - -// parityChainSpecAltBnConstOperationPricing defines the price -// policy for bn const operation(used after istanbul) -type parityChainSpecAltBnConstOperationPricing struct { - Price uint64 `json:"price"` -} - -// parityChainSepcAltBnPairingPricing defines the price policy -// for bn pairing. -type parityChainSepcAltBnPairingPricing struct { - Base uint64 `json:"base"` - Pair uint64 `json:"pair"` -} - -// parityChainSpecBlakePricing defines the price policy for blake2 f -// compression. -type parityChainSpecBlakePricing struct { - GasPerRound uint64 `json:"gas_per_round"` -} - -type parityChainSpecAlternativePrice struct { - AltBnConstOperationPrice *parityChainSpecAltBnConstOperationPricing `json:"alt_bn128_const_operations,omitempty"` - AltBnPairingPrice *parityChainSepcAltBnPairingPricing `json:"alt_bn128_pairing,omitempty"` -} - -// parityChainSpecVersionedPricing represents a single version price policy. -type parityChainSpecVersionedPricing struct { - Price *parityChainSpecAlternativePrice `json:"price,omitempty"` - Info string `json:"info,omitempty"` -} - -// newParityChainSpec converts a go-ethereum genesis block into a Parity specific -// chain specification format. -func newParityChainSpec(network string, genesis *core.Genesis, bootnodes []string) (*parityChainSpec, error) { - // Only ethash is currently supported between go-ethereum and Parity - if genesis.Config.Ethash == nil { - return nil, errors.New("unsupported consensus engine") - } - // Reconstruct the chain spec in Parity's format - spec := &parityChainSpec{ - Name: network, - Nodes: bootnodes, - Datadir: strings.ToLower(network), - } - spec.Engine.Ethash.Params.BlockReward = make(map[string]string) - spec.Engine.Ethash.Params.DifficultyBombDelays = make(map[string]string) - // Frontier - spec.Engine.Ethash.Params.MinimumDifficulty = (*hexutil.Big)(params.MinimumDifficulty) - spec.Engine.Ethash.Params.DifficultyBoundDivisor = (*hexutil.Big)(params.DifficultyBoundDivisor) - spec.Engine.Ethash.Params.DurationLimit = (*hexutil.Big)(params.DurationLimit) - spec.Engine.Ethash.Params.BlockReward["0x0"] = hexutil.EncodeBig(ethash.FrontierBlockReward) - - // Homestead - spec.Engine.Ethash.Params.HomesteadTransition = hexutil.Uint64(genesis.Config.HomesteadBlock.Uint64()) - - // Tangerine Whistle : 150 - // https://github.com/ethereum/EIPs/blob/master/EIPS/eip-608.md - spec.Params.EIP150Transition = hexutil.Uint64(genesis.Config.EIP150Block.Uint64()) - - // Spurious Dragon: 155, 160, 161, 170 - // https://github.com/ethereum/EIPs/blob/master/EIPS/eip-607.md - spec.Params.EIP155Transition = hexutil.Uint64(genesis.Config.EIP155Block.Uint64()) - spec.Params.EIP160Transition = hexutil.Uint64(genesis.Config.EIP155Block.Uint64()) - spec.Params.EIP161abcTransition = hexutil.Uint64(genesis.Config.EIP158Block.Uint64()) - spec.Params.EIP161dTransition = hexutil.Uint64(genesis.Config.EIP158Block.Uint64()) - - // Byzantium - if num := genesis.Config.ByzantiumBlock; num != nil { - spec.setByzantium(num) - } - // Constantinople - if num := genesis.Config.ConstantinopleBlock; num != nil { - spec.setConstantinople(num) - } - // ConstantinopleFix (remove eip-1283) - if num := genesis.Config.PetersburgBlock; num != nil { - spec.setConstantinopleFix(num) - } - // Istanbul - if num := genesis.Config.IstanbulBlock; num != nil { - spec.setIstanbul(num) - } - spec.Params.MaximumExtraDataSize = (hexutil.Uint64)(params.MaximumExtraDataSize) - spec.Params.MinGasLimit = (hexutil.Uint64)(params.MinGasLimit) - spec.Params.GasLimitBoundDivisor = (math2.HexOrDecimal64)(params.GasLimitBoundDivisor) - spec.Params.NetworkID = (hexutil.Uint64)(genesis.Config.ChainID.Uint64()) - spec.Params.ChainID = (hexutil.Uint64)(genesis.Config.ChainID.Uint64()) - spec.Params.MaxCodeSize = params.MaxCodeSize - // geth has it set from zero - spec.Params.MaxCodeSizeTransition = 0 - - // Disable this one - spec.Params.EIP98Transition = math.MaxInt64 - - spec.Genesis.Seal.Ethereum.Nonce = types.EncodeNonce(genesis.Nonce) - spec.Genesis.Seal.Ethereum.MixHash = genesis.Mixhash[:] - spec.Genesis.Difficulty = (*hexutil.Big)(genesis.Difficulty) - spec.Genesis.Author = genesis.Coinbase - spec.Genesis.Timestamp = (hexutil.Uint64)(genesis.Timestamp) - spec.Genesis.ParentHash = genesis.ParentHash - spec.Genesis.ExtraData = genesis.ExtraData - spec.Genesis.GasLimit = (hexutil.Uint64)(genesis.GasLimit) - - spec.Accounts = make(map[common.UnprefixedAddress]*parityChainSpecAccount) - for address, account := range genesis.Alloc { - bal := math2.HexOrDecimal256(*account.Balance) - - spec.Accounts[common.UnprefixedAddress(address)] = &parityChainSpecAccount{ - Balance: bal, - Nonce: math2.HexOrDecimal64(account.Nonce), - } - } - spec.setPrecompile(1, &parityChainSpecBuiltin{Name: "ecrecover", - Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 3000}}}) - - spec.setPrecompile(2, &parityChainSpecBuiltin{ - Name: "sha256", Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 60, Word: 12}}, - }) - spec.setPrecompile(3, &parityChainSpecBuiltin{ - Name: "ripemd160", Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 600, Word: 120}}, - }) - spec.setPrecompile(4, &parityChainSpecBuiltin{ - Name: "identity", Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 15, Word: 3}}, - }) - if genesis.Config.ByzantiumBlock != nil { - spec.setPrecompile(5, &parityChainSpecBuiltin{ - Name: "modexp", - ActivateAt: (*hexutil.Big)(genesis.Config.ByzantiumBlock), - Pricing: &parityChainSpecPricing{ - ModExp: &parityChainSpecModExpPricing{Divisor: 20}, - }, - }) - spec.setPrecompile(6, &parityChainSpecBuiltin{ - Name: "alt_bn128_add", - ActivateAt: (*hexutil.Big)(genesis.Config.ByzantiumBlock), - Pricing: &parityChainSpecPricing{ - Linear: &parityChainSpecLinearPricing{Base: 500, Word: 0}, - }, - }) - spec.setPrecompile(7, &parityChainSpecBuiltin{ - Name: "alt_bn128_mul", - ActivateAt: (*hexutil.Big)(genesis.Config.ByzantiumBlock), - Pricing: &parityChainSpecPricing{ - Linear: &parityChainSpecLinearPricing{Base: 40000, Word: 0}, - }, - }) - spec.setPrecompile(8, &parityChainSpecBuiltin{ - Name: "alt_bn128_pairing", - ActivateAt: (*hexutil.Big)(genesis.Config.ByzantiumBlock), - Pricing: &parityChainSpecPricing{ - AltBnPairing: &parityChainSepcAltBnPairingPricing{Base: 100000, Pair: 80000}, - }, - }) - } - if genesis.Config.IstanbulBlock != nil { - if genesis.Config.ByzantiumBlock == nil { - return nil, errors.New("invalid genesis, istanbul fork is enabled while byzantium is not") - } - spec.setPrecompile(6, &parityChainSpecBuiltin{ - Name: "alt_bn128_add", - ActivateAt: (*hexutil.Big)(genesis.Config.ByzantiumBlock), - Pricing: map[*hexutil.Big]*parityChainSpecVersionedPricing{ - (*hexutil.Big)(big.NewInt(0)): { - Price: &parityChainSpecAlternativePrice{ - AltBnConstOperationPrice: &parityChainSpecAltBnConstOperationPricing{Price: 500}, - }, - }, - (*hexutil.Big)(genesis.Config.IstanbulBlock): { - Price: &parityChainSpecAlternativePrice{ - AltBnConstOperationPrice: &parityChainSpecAltBnConstOperationPricing{Price: 150}, - }, - }, - }, - }) - spec.setPrecompile(7, &parityChainSpecBuiltin{ - Name: "alt_bn128_mul", - ActivateAt: (*hexutil.Big)(genesis.Config.ByzantiumBlock), - Pricing: map[*hexutil.Big]*parityChainSpecVersionedPricing{ - (*hexutil.Big)(big.NewInt(0)): { - Price: &parityChainSpecAlternativePrice{ - AltBnConstOperationPrice: &parityChainSpecAltBnConstOperationPricing{Price: 40000}, - }, - }, - (*hexutil.Big)(genesis.Config.IstanbulBlock): { - Price: &parityChainSpecAlternativePrice{ - AltBnConstOperationPrice: &parityChainSpecAltBnConstOperationPricing{Price: 6000}, - }, - }, - }, - }) - spec.setPrecompile(8, &parityChainSpecBuiltin{ - Name: "alt_bn128_pairing", - ActivateAt: (*hexutil.Big)(genesis.Config.ByzantiumBlock), - Pricing: map[*hexutil.Big]*parityChainSpecVersionedPricing{ - (*hexutil.Big)(big.NewInt(0)): { - Price: &parityChainSpecAlternativePrice{ - AltBnPairingPrice: &parityChainSepcAltBnPairingPricing{Base: 100000, Pair: 80000}, - }, - }, - (*hexutil.Big)(genesis.Config.IstanbulBlock): { - Price: &parityChainSpecAlternativePrice{ - AltBnPairingPrice: &parityChainSepcAltBnPairingPricing{Base: 45000, Pair: 34000}, - }, - }, - }, - }) - spec.setPrecompile(9, &parityChainSpecBuiltin{ - Name: "blake2_f", - ActivateAt: (*hexutil.Big)(genesis.Config.IstanbulBlock), - Pricing: &parityChainSpecPricing{ - Blake2F: &parityChainSpecBlakePricing{GasPerRound: 1}, - }, - }) - } - return spec, nil -} - -func (spec *parityChainSpec) setPrecompile(address byte, data *parityChainSpecBuiltin) { - if spec.Accounts == nil { - spec.Accounts = make(map[common.UnprefixedAddress]*parityChainSpecAccount) - } - a := common.UnprefixedAddress(common.BytesToAddress([]byte{address})) - if _, exist := spec.Accounts[a]; !exist { - spec.Accounts[a] = &parityChainSpecAccount{} - } - spec.Accounts[a].Builtin = data -} - -func (spec *parityChainSpec) setByzantium(num *big.Int) { - spec.Engine.Ethash.Params.BlockReward[hexutil.EncodeBig(num)] = hexutil.EncodeBig(ethash.ByzantiumBlockReward) - spec.Engine.Ethash.Params.DifficultyBombDelays[hexutil.EncodeBig(num)] = hexutil.EncodeUint64(3000000) - n := hexutil.Uint64(num.Uint64()) - spec.Engine.Ethash.Params.EIP100bTransition = n - spec.Params.EIP140Transition = n - spec.Params.EIP211Transition = n - spec.Params.EIP214Transition = n - spec.Params.EIP658Transition = n -} - -func (spec *parityChainSpec) setConstantinople(num *big.Int) { - spec.Engine.Ethash.Params.BlockReward[hexutil.EncodeBig(num)] = hexutil.EncodeBig(ethash.ConstantinopleBlockReward) - spec.Engine.Ethash.Params.DifficultyBombDelays[hexutil.EncodeBig(num)] = hexutil.EncodeUint64(2000000) - n := hexutil.Uint64(num.Uint64()) - spec.Params.EIP145Transition = n - spec.Params.EIP1014Transition = n - spec.Params.EIP1052Transition = n - spec.Params.EIP1283Transition = n -} - -func (spec *parityChainSpec) setConstantinopleFix(num *big.Int) { - spec.Params.EIP1283DisableTransition = hexutil.Uint64(num.Uint64()) -} - -func (spec *parityChainSpec) setIstanbul(num *big.Int) { - spec.Params.EIP1344Transition = hexutil.Uint64(num.Uint64()) - spec.Params.EIP1884Transition = hexutil.Uint64(num.Uint64()) - spec.Params.EIP2028Transition = hexutil.Uint64(num.Uint64()) - spec.Params.EIP1283ReenableTransition = hexutil.Uint64(num.Uint64()) -} - -// pyEthereumGenesisSpec represents the genesis specification format used by the -// Python Ethereum implementation. -type pyEthereumGenesisSpec struct { - Nonce types.BlockNonce `json:"nonce"` - Timestamp hexutil.Uint64 `json:"timestamp"` - ExtraData hexutil.Bytes `json:"extraData"` - GasLimit hexutil.Uint64 `json:"gasLimit"` - Difficulty *hexutil.Big `json:"difficulty"` - Mixhash common.Hash `json:"mixhash"` - Coinbase common.Address `json:"coinbase"` - Alloc core.GenesisAlloc `json:"alloc"` - ParentHash common.Hash `json:"parentHash"` -} - -// newPyEthereumGenesisSpec converts a go-ethereum genesis block into a Parity specific -// chain specification format. -func newPyEthereumGenesisSpec(network string, genesis *core.Genesis) (*pyEthereumGenesisSpec, error) { - // Only ethash is currently supported between go-ethereum and pyethereum - if genesis.Config.Ethash == nil { - return nil, errors.New("unsupported consensus engine") - } - spec := &pyEthereumGenesisSpec{ - Nonce: types.EncodeNonce(genesis.Nonce), - Timestamp: (hexutil.Uint64)(genesis.Timestamp), - ExtraData: genesis.ExtraData, - GasLimit: (hexutil.Uint64)(genesis.GasLimit), - Difficulty: (*hexutil.Big)(genesis.Difficulty), - Mixhash: genesis.Mixhash, - Coinbase: genesis.Coinbase, - Alloc: genesis.Alloc, - ParentHash: genesis.ParentHash, - } - return spec, nil -} diff --git a/cmd/puppeth/genesis_test.go b/cmd/puppeth/genesis_test.go deleted file mode 100644 index 605c1070a80cb..0000000000000 --- a/cmd/puppeth/genesis_test.go +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright 2018 The go-ethereum Authors -// This file is part of go-ethereum. -// -// go-ethereum is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// go-ethereum is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with go-ethereum. If not, see . - -package main - -import ( - "bytes" - "encoding/json" - "os" - "reflect" - "strings" - "testing" - - "github.com/davecgh/go-spew/spew" - "github.com/ethereum/go-ethereum/core" -) - -// Tests the go-ethereum to Aleth chainspec conversion for the Stureby testnet. -func TestAlethSturebyConverter(t *testing.T) { - blob, err := os.ReadFile("testdata/stureby_geth.json") - if err != nil { - t.Fatalf("could not read file: %v", err) - } - var genesis core.Genesis - if err := json.Unmarshal(blob, &genesis); err != nil { - t.Fatalf("failed parsing genesis: %v", err) - } - spec, err := newAlethGenesisSpec("stureby", &genesis) - if err != nil { - t.Fatalf("failed creating chainspec: %v", err) - } - - expBlob, err := os.ReadFile("testdata/stureby_aleth.json") - if err != nil { - t.Fatalf("could not read file: %v", err) - } - expspec := &alethGenesisSpec{} - if err := json.Unmarshal(expBlob, expspec); err != nil { - t.Fatalf("failed parsing genesis: %v", err) - } - if !reflect.DeepEqual(expspec, spec) { - t.Errorf("chainspec mismatch") - c := spew.ConfigState{ - DisablePointerAddresses: true, - SortKeys: true, - } - exp := strings.Split(c.Sdump(expspec), "\n") - got := strings.Split(c.Sdump(spec), "\n") - for i := 0; i < len(exp) && i < len(got); i++ { - if exp[i] != got[i] { - t.Logf("got: %v\nexp: %v\n", exp[i], got[i]) - } - } - } -} - -// Tests the go-ethereum to Parity chainspec conversion for the Stureby testnet. -func TestParitySturebyConverter(t *testing.T) { - blob, err := os.ReadFile("testdata/stureby_geth.json") - if err != nil { - t.Fatalf("could not read file: %v", err) - } - var genesis core.Genesis - if err := json.Unmarshal(blob, &genesis); err != nil { - t.Fatalf("failed parsing genesis: %v", err) - } - spec, err := newParityChainSpec("stureby", &genesis, []string{}) - if err != nil { - t.Fatalf("failed creating chainspec: %v", err) - } - enc, err := json.MarshalIndent(spec, "", " ") - if err != nil { - t.Fatalf("failed encoding chainspec: %v", err) - } - expBlob, err := os.ReadFile("testdata/stureby_parity.json") - if err != nil { - t.Fatalf("could not read file: %v", err) - } - if !bytes.Equal(expBlob, enc) { - t.Fatalf("chainspec mismatch") - } -} diff --git a/cmd/puppeth/module.go b/cmd/puppeth/module.go index b6a029a01a48a..771ae38058bc9 100644 --- a/cmd/puppeth/module.go +++ b/cmd/puppeth/module.go @@ -150,3 +150,12 @@ func checkPort(host string, port int) error { conn.Close() return nil } + +// getEthName gets the Ethereum Name from ethstats +func getEthName(s string) string { + n := strings.Index(s, ":") + if n >= 0 { + return s[:n] + } + return s +} diff --git a/cmd/puppeth/module_dashboard.go b/cmd/puppeth/module_dashboard.go index 35cfada66fd3f..fbbbb66501a7d 100644 --- a/cmd/puppeth/module_dashboard.go +++ b/cmd/puppeth/module_dashboard.go @@ -18,7 +18,6 @@ package main import ( "bytes" - "encoding/json" "fmt" "html/template" "math/rand" @@ -582,36 +581,6 @@ func deployDashboard(client *sshClient, network string, conf *config, config *da // Marshal the genesis spec files for go-ethereum and all the other clients genesis, _ := conf.Genesis.MarshalJSON() files[filepath.Join(workdir, network+".json")] = genesis - - if conf.Genesis.Config.Ethash != nil { - cppSpec, err := newAlethGenesisSpec(network, conf.Genesis) - if err != nil { - return nil, err - } - cppSpecJSON, _ := json.Marshal(cppSpec) - files[filepath.Join(workdir, network+"-cpp.json")] = cppSpecJSON - - harmonySpecJSON, _ := conf.Genesis.MarshalJSON() - files[filepath.Join(workdir, network+"-harmony.json")] = harmonySpecJSON - - paritySpec, err := newParityChainSpec(network, conf.Genesis, conf.bootnodes) - if err != nil { - return nil, err - } - paritySpecJSON, _ := json.Marshal(paritySpec) - files[filepath.Join(workdir, network+"-parity.json")] = paritySpecJSON - - pyethSpec, err := newPyEthereumGenesisSpec(network, conf.Genesis) - if err != nil { - return nil, err - } - pyethSpecJSON, _ := json.Marshal(pyethSpec) - files[filepath.Join(workdir, network+"-python.json")] = pyethSpecJSON - } else { - for _, client := range []string{"cpp", "harmony", "parity", "python"} { - files[filepath.Join(workdir, network+"-"+client+".json")] = []byte{} - } - } files[filepath.Join(workdir, "puppeth.png")] = dashboardMascot // Upload the deployment files to the remote server (and clean up afterwards) diff --git a/cmd/puppeth/module_explorer.go b/cmd/puppeth/module_explorer.go index 1165f70fcf516..3812f9fdb9637 100644 --- a/cmd/puppeth/module_explorer.go +++ b/cmd/puppeth/module_explorer.go @@ -104,7 +104,7 @@ func deployExplorer(client *sshClient, network string, bootnodes []string, confi "Datadir": config.node.datadir, "DBDir": config.dbdir, "EthPort": config.node.port, - "EthName": config.node.ethstats[:strings.Index(config.node.ethstats, ":")], + "EthName": getEthName(config.node.ethstats), "WebPort": config.port, "Transformer": transformer, }) diff --git a/cmd/puppeth/module_faucet.go b/cmd/puppeth/module_faucet.go index 88cb80ae4c42c..a4f6e65694df4 100644 --- a/cmd/puppeth/module_faucet.go +++ b/cmd/puppeth/module_faucet.go @@ -116,7 +116,7 @@ func deployFaucet(client *sshClient, network string, bootnodes []string, config "VHost": config.host, "ApiPort": config.port, "EthPort": config.node.port, - "EthName": config.node.ethstats[:strings.Index(config.node.ethstats, ":")], + "EthName": getEthName(config.node.ethstats), "CaptchaToken": config.captchaToken, "CaptchaSecret": config.captchaSecret, "FaucetAmount": config.amount, diff --git a/cmd/puppeth/module_node.go b/cmd/puppeth/module_node.go index 3ea96870d4f5d..b8aa30db39a8d 100644 --- a/cmd/puppeth/module_node.go +++ b/cmd/puppeth/module_node.go @@ -123,7 +123,7 @@ func deployNode(client *sshClient, network string, bootnodes []string, config *n "TotalPeers": config.peersTotal, "Light": config.peersLight > 0, "LightPeers": config.peersLight, - "Ethstats": config.ethstats[:strings.Index(config.ethstats, ":")], + "Ethstats": getEthName(config.ethstats), "Etherbase": config.etherbase, "GasTarget": config.gasTarget, "GasLimit": config.gasLimit, diff --git a/cmd/puppeth/puppeth.go b/cmd/puppeth/puppeth.go index c3de5f9360242..415542b60cc9f 100644 --- a/cmd/puppeth/puppeth.go +++ b/cmd/puppeth/puppeth.go @@ -24,7 +24,7 @@ import ( "time" "github.com/ethereum/go-ethereum/log" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) // main is just a boring entry point to set up the CLI app. @@ -33,11 +33,11 @@ func main() { app.Name = "puppeth" app.Usage = "assemble and maintain private Ethereum networks" app.Flags = []cli.Flag{ - cli.StringFlag{ + &cli.StringFlag{ Name: "network", Usage: "name of the network to administer (no spaces or hyphens, please)", }, - cli.IntFlag{ + &cli.IntFlag{ Name: "loglevel", Value: 3, Usage: "log level to emit to the screen", diff --git a/cmd/puppeth/ssh.go b/cmd/puppeth/ssh.go index 95a36f327236b..a20b3bfda2095 100644 --- a/cmd/puppeth/ssh.go +++ b/cmd/puppeth/ssh.go @@ -30,7 +30,7 @@ import ( "github.com/ethereum/go-ethereum/log" "golang.org/x/crypto/ssh" "golang.org/x/crypto/ssh/agent" - "golang.org/x/crypto/ssh/terminal" + "golang.org/x/term" ) // sshClient is a small wrapper around Go's SSH client with a few utility methods @@ -101,7 +101,7 @@ func dial(server string, pubkey []byte) (*sshClient, error) { key, err := ssh.ParsePrivateKey(buf) if err != nil { fmt.Printf("What's the decryption password for %s? (won't be echoed)\n>", path) - blob, err := terminal.ReadPassword(int(os.Stdin.Fd())) + blob, err := term.ReadPassword(int(os.Stdin.Fd())) fmt.Println() if err != nil { log.Warn("Couldn't read password", "err", err) @@ -118,7 +118,7 @@ func dial(server string, pubkey []byte) (*sshClient, error) { } auths = append(auths, ssh.PasswordCallback(func() (string, error) { fmt.Printf("What's the login password for %s at %s? (won't be echoed)\n> ", username, server) - blob, err := terminal.ReadPassword(int(os.Stdin.Fd())) + blob, err := term.ReadPassword(int(os.Stdin.Fd())) fmt.Println() return string(blob), err @@ -163,7 +163,7 @@ func dial(server string, pubkey []byte) (*sshClient, error) { return nil } // We have a mismatch, forbid connecting - return errors.New("ssh key mismatch, readd the machine to update") + return errors.New("ssh key mismatch, re-add the machine to update") } client, err := ssh.Dial("tcp", hostport, &ssh.ClientConfig{User: username, Auth: auths, HostKeyCallback: keycheck}) if err != nil { diff --git a/cmd/puppeth/wizard.go b/cmd/puppeth/wizard.go index f7aafd4dd90a9..6e5ca41d68fab 100644 --- a/cmd/puppeth/wizard.go +++ b/cmd/puppeth/wizard.go @@ -34,7 +34,7 @@ import ( "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/log" "github.com/peterh/liner" - "golang.org/x/crypto/ssh/terminal" + "golang.org/x/term" ) // config contains all the configurations needed by puppeth that should be saved @@ -228,7 +228,7 @@ func (w *wizard) readDefaultFloat(def float64) float64 { // line and returns it. The input will not be echoed. func (w *wizard) readPassword() string { fmt.Printf("> ") - text, err := terminal.ReadPassword(int(os.Stdin.Fd())) + text, err := term.ReadPassword(int(os.Stdin.Fd())) if err != nil { log.Crit("Failed to read password", "err", err) } diff --git a/cmd/puppeth/wizard_genesis.go b/cmd/puppeth/wizard_genesis.go index cb056ab13356a..ac17bc7b271cc 100644 --- a/cmd/puppeth/wizard_genesis.go +++ b/cmd/puppeth/wizard_genesis.go @@ -250,8 +250,8 @@ func (w *wizard) manageGenesis() { case "2": // Save whatever genesis configuration we currently have fmt.Println() - fmt.Printf("Which folder to save the genesis specs into? (default = current)\n") - fmt.Printf(" Will create %s.json, %s-aleth.json, %s-harmony.json, %s-parity.json\n", w.network, w.network, w.network, w.network) + fmt.Printf("Which folder to save the genesis spec into? (default = current)\n") + fmt.Printf(" Will create %s.json\n", w.network) folder := w.readDefaultString(".") if err := os.MkdirAll(folder, 0755); err != nil { @@ -268,21 +268,6 @@ func (w *wizard) manageGenesis() { } log.Info("Saved native genesis chain spec", "path", gethJson) - // Export the genesis spec used by Aleth (formerly C++ Ethereum) - if spec, err := newAlethGenesisSpec(w.network, w.conf.Genesis); err != nil { - log.Error("Failed to create Aleth chain spec", "err", err) - } else { - saveGenesis(folder, w.network, "aleth", spec) - } - // Export the genesis spec used by Parity - if spec, err := newParityChainSpec(w.network, w.conf.Genesis, []string{}); err != nil { - log.Error("Failed to create Parity chain spec", "err", err) - } else { - saveGenesis(folder, w.network, "parity", spec) - } - // Export the genesis spec used by Harmony (formerly EthereumJ) - saveGenesis(folder, w.network, "harmony", w.conf.Genesis) - case "3": // Make sure we don't have any services running if len(w.conf.servers()) > 0 { @@ -298,15 +283,3 @@ func (w *wizard) manageGenesis() { return } } - -// saveGenesis JSON encodes an arbitrary genesis spec into a pre-defined file. -func saveGenesis(folder, network, client string, spec interface{}) { - path := filepath.Join(folder, fmt.Sprintf("%s-%s.json", network, client)) - - out, _ := json.MarshalIndent(spec, "", " ") - if err := os.WriteFile(path, out, 0644); err != nil { - log.Error("Failed to save genesis file", "client", client, "err", err) - return - } - log.Info("Saved genesis chain spec", "client", client, "path", path) -} diff --git a/cmd/rlpdump/main.go b/cmd/rlpdump/main.go index 9c0af012480fe..70337749aea30 100644 --- a/cmd/rlpdump/main.go +++ b/cmd/rlpdump/main.go @@ -83,7 +83,7 @@ func main() { if err != nil { die(err) } - fmt.Printf("0x%x\n", data) + fmt.Printf("%#x\n", data) return } else { err := rlpToText(r, out) diff --git a/cmd/rlpdump/rlpdump_test.go b/cmd/rlpdump/rlpdump_test.go index 899beef32f4a6..a9ab57fdb880e 100644 --- a/cmd/rlpdump/rlpdump_test.go +++ b/cmd/rlpdump/rlpdump_test.go @@ -43,7 +43,7 @@ func TestRoundtrip(t *testing.T) { t.Errorf("test %d: error %v", i, err) continue } - have := fmt.Sprintf("0x%x", rlpBytes) + have := fmt.Sprintf("%#x", rlpBytes) if have != want { t.Errorf("test %d: have\n%v\nwant:\n%v\n", i, have, want) } diff --git a/cmd/utils/cmd.go b/cmd/utils/cmd.go index 47ad3b22c8dd1..90f0090414773 100644 --- a/cmd/utils/cmd.go +++ b/cmd/utils/cmd.go @@ -41,7 +41,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/rlp" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) const ( @@ -78,10 +78,10 @@ func StartNode(ctx *cli.Context, stack *node.Node, isConsole bool) { defer signal.Stop(sigc) minFreeDiskSpace := 2 * ethconfig.Defaults.TrieDirtyCache // Default 2 * 256Mb - if ctx.GlobalIsSet(MinFreeDiskSpaceFlag.Name) { - minFreeDiskSpace = ctx.GlobalInt(MinFreeDiskSpaceFlag.Name) - } else if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheGCFlag.Name) { - minFreeDiskSpace = 2 * ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100 + if ctx.IsSet(MinFreeDiskSpaceFlag.Name) { + minFreeDiskSpace = ctx.Int(MinFreeDiskSpaceFlag.Name) + } else if ctx.IsSet(CacheFlag.Name) || ctx.IsSet(CacheGCFlag.Name) { + minFreeDiskSpace = 2 * ctx.Int(CacheFlag.Name) * ctx.Int(CacheGCFlag.Name) / 100 } if minFreeDiskSpace > 0 { go monitorFreeDiskSpace(sigc, stack.InstanceDir(), uint64(minFreeDiskSpace)*1024*1024) diff --git a/cmd/utils/customflags.go b/cmd/utils/customflags.go deleted file mode 100644 index e5be085a5db7e..0000000000000 --- a/cmd/utils/customflags.go +++ /dev/null @@ -1,211 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of go-ethereum. -// -// go-ethereum is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// go-ethereum is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with go-ethereum. If not, see . - -package utils - -import ( - "encoding" - "errors" - "flag" - "math/big" - "os" - "os/user" - "path" - "strings" - - "github.com/ethereum/go-ethereum/common/math" - "gopkg.in/urfave/cli.v1" -) - -// Custom type which is registered in the flags library which cli uses for -// argument parsing. This allows us to expand Value to an absolute path when -// the argument is parsed -type DirectoryString string - -func (s *DirectoryString) String() string { - return string(*s) -} - -func (s *DirectoryString) Set(value string) error { - *s = DirectoryString(expandPath(value)) - return nil -} - -// Custom cli.Flag type which expand the received string to an absolute path. -// e.g. ~/.ethereum -> /home/username/.ethereum -type DirectoryFlag struct { - Name string - Value DirectoryString - Usage string - EnvVar string -} - -func (f DirectoryFlag) String() string { - return cli.FlagStringer(f) -} - -// called by cli library, grabs variable from environment (if in env) -// and adds variable to flag set for parsing. -func (f DirectoryFlag) Apply(set *flag.FlagSet) { - eachName(f.Name, func(name string) { - set.Var(&f.Value, f.Name, f.Usage) - }) -} - -func (f DirectoryFlag) GetName() string { - return f.Name -} - -func (f *DirectoryFlag) Set(value string) { - f.Value.Set(value) -} - -func eachName(longName string, fn func(string)) { - parts := strings.Split(longName, ",") - for _, name := range parts { - name = strings.Trim(name, " ") - fn(name) - } -} - -type TextMarshaler interface { - encoding.TextMarshaler - encoding.TextUnmarshaler -} - -// textMarshalerVal turns a TextMarshaler into a flag.Value -type textMarshalerVal struct { - v TextMarshaler -} - -func (v textMarshalerVal) String() string { - if v.v == nil { - return "" - } - text, _ := v.v.MarshalText() - return string(text) -} - -func (v textMarshalerVal) Set(s string) error { - return v.v.UnmarshalText([]byte(s)) -} - -// TextMarshalerFlag wraps a TextMarshaler value. -type TextMarshalerFlag struct { - Name string - Value TextMarshaler - Usage string - EnvVar string -} - -func (f TextMarshalerFlag) GetName() string { - return f.Name -} - -func (f TextMarshalerFlag) String() string { - return cli.FlagStringer(f) -} - -func (f TextMarshalerFlag) Apply(set *flag.FlagSet) { - eachName(f.Name, func(name string) { - set.Var(textMarshalerVal{f.Value}, f.Name, f.Usage) - }) -} - -// GlobalTextMarshaler returns the value of a TextMarshalerFlag from the global flag set. -func GlobalTextMarshaler(ctx *cli.Context, name string) TextMarshaler { - val := ctx.GlobalGeneric(name) - if val == nil { - return nil - } - return val.(textMarshalerVal).v -} - -// BigFlag is a command line flag that accepts 256 bit big integers in decimal or -// hexadecimal syntax. -type BigFlag struct { - Name string - Value *big.Int - Usage string - EnvVar string -} - -// bigValue turns *big.Int into a flag.Value -type bigValue big.Int - -func (b *bigValue) String() string { - if b == nil { - return "" - } - return (*big.Int)(b).String() -} - -func (b *bigValue) Set(s string) error { - intVal, ok := math.ParseBig256(s) - if !ok { - return errors.New("invalid integer syntax") - } - *b = (bigValue)(*intVal) - return nil -} - -func (f BigFlag) GetName() string { - return f.Name -} - -func (f BigFlag) String() string { - return cli.FlagStringer(f) -} - -func (f BigFlag) Apply(set *flag.FlagSet) { - eachName(f.Name, func(name string) { - f.Value = new(big.Int) - set.Var((*bigValue)(f.Value), f.Name, f.Usage) - }) -} - -// GlobalBig returns the value of a BigFlag from the global flag set. -func GlobalBig(ctx *cli.Context, name string) *big.Int { - val := ctx.GlobalGeneric(name) - if val == nil { - return nil - } - return (*big.Int)(val.(*bigValue)) -} - -// Expands a file path -// 1. replace tilde with users home dir -// 2. expands embedded environment variables -// 3. cleans the path, e.g. /a/b/../c -> /a/c -// Note, it has limitations, e.g. ~someuser/tmp will not be expanded -func expandPath(p string) string { - if strings.HasPrefix(p, "~/") || strings.HasPrefix(p, "~\\") { - if home := HomeDir(); home != "" { - p = home + p[1:] - } - } - return path.Clean(os.ExpandEnv(p)) -} - -func HomeDir() string { - if home := os.Getenv("HOME"); home != "" { - return home - } - if usr, err := user.Current(); err == nil { - return usr.HomeDir - } - return "" -} diff --git a/cmd/utils/diskusage.go b/cmd/utils/diskusage.go index 14cd5cd0bef86..0e88f91944302 100644 --- a/cmd/utils/diskusage.go +++ b/cmd/utils/diskusage.go @@ -33,6 +33,7 @@ func getFreeDiskSpace(path string) (uint64, error) { // Available blocks * size per block = available space in bytes var bavail = stat.Bavail + // nolint:staticcheck if stat.Bavail < 0 { // FreeBSD can have a negative number of blocks available // because of the grace limit. diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 4ce16ef90031f..9e95193343a95 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -20,7 +20,6 @@ package utils import ( "crypto/ecdsa" "fmt" - "io" "math" "math/big" "os" @@ -28,8 +27,6 @@ import ( godebug "runtime/debug" "strconv" "strings" - "text/tabwriter" - "text/template" "time" "github.com/ethereum/go-ethereum/accounts" @@ -46,6 +43,7 @@ import ( ethcatalyst "github.com/ethereum/go-ethereum/eth/catalyst" "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/ethconfig" + "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/eth/gasprice" "github.com/ethereum/go-ethereum/eth/tracers" "github.com/ethereum/go-ethereum/ethdb" @@ -67,39 +65,12 @@ import ( "github.com/ethereum/go-ethereum/p2p/nat" "github.com/ethereum/go-ethereum/p2p/netutil" "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/rpc" pcsclite "github.com/gballet/go-libpcsclite" gopsutil "github.com/shirou/gopsutil/mem" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) -func init() { - cli.AppHelpTemplate = `{{.Name}} {{if .Flags}}[global options] {{end}}command{{if .Flags}} [command options]{{end}} [arguments...] - -VERSION: - {{.Version}} - -COMMANDS: - {{range .Commands}}{{.Name}}{{with .ShortName}}, {{.}}{{end}}{{ "\t" }}{{.Usage}} - {{end}}{{if .Flags}} -GLOBAL OPTIONS: - {{range .Flags}}{{.}} - {{end}}{{end}} -` - cli.CommandHelpTemplate = flags.CommandHelpTemplate - cli.HelpPrinter = printHelp -} - -func printHelp(out io.Writer, templ string, data interface{}) { - funcMap := template.FuncMap{"join": strings.Join} - t := template.Must(template.New("help").Funcs(funcMap).Parse(templ)) - w := tabwriter.NewWriter(out, 38, 8, 2, ' ', 0) - err := t.Execute(w, data) - if err != nil { - panic(err) - } - w.Flush() -} - // These are all the command line flags we support. // If you add to this list, please remember to include the // flag in the appropriate command definition. @@ -109,721 +80,902 @@ func printHelp(out io.Writer, templ string, data interface{}) { var ( // General settings - DataDirFlag = DirectoryFlag{ - Name: "datadir", - Usage: "Data directory for the databases and keystore", - Value: DirectoryString(node.DefaultDataDir()), - } - RemoteDBFlag = cli.StringFlag{ - Name: "remotedb", - Usage: "URL for remote database", - } - AncientFlag = DirectoryFlag{ - Name: "datadir.ancient", - Usage: "Data directory for ancient chain segments (default = inside chaindata)", - } - MinFreeDiskSpaceFlag = DirectoryFlag{ - Name: "datadir.minfreedisk", - Usage: "Minimum free disk space in MB, once reached triggers auto shut down (default = --cache.gc converted to MB, 0 = disabled)", - } - KeyStoreDirFlag = DirectoryFlag{ - Name: "keystore", - Usage: "Directory for the keystore (default = inside the datadir)", - } - USBFlag = cli.BoolFlag{ - Name: "usb", - Usage: "Enable monitoring and management of USB hardware wallets", - } - SmartCardDaemonPathFlag = cli.StringFlag{ - Name: "pcscdpath", - Usage: "Path to the smartcard daemon (pcscd) socket file", - Value: pcsclite.PCSCDSockName, - } - NetworkIdFlag = cli.Uint64Flag{ - Name: "networkid", - Usage: "Explicitly set network id (integer)(For testnets: use --ropsten, --rinkeby, --goerli instead)", - Value: ethconfig.Defaults.NetworkId, - } - MainnetFlag = cli.BoolFlag{ - Name: "mainnet", - Usage: "Ethereum mainnet", - } - RopstenFlag = cli.BoolFlag{ - Name: "ropsten", - Usage: "Ropsten network: pre-configured proof-of-work test network", - } - RinkebyFlag = cli.BoolFlag{ - Name: "rinkeby", - Usage: "Rinkeby network: pre-configured proof-of-authority test network", - } - GoerliFlag = cli.BoolFlag{ - Name: "goerli", - Usage: "Görli network: pre-configured proof-of-authority test network", - } - SepoliaFlag = cli.BoolFlag{ - Name: "sepolia", - Usage: "Sepolia network: pre-configured proof-of-work test network", - } - KilnFlag = cli.BoolFlag{ - Name: "kiln", - Usage: "Kiln network: pre-configured proof-of-work to proof-of-stake test network", - } - DeveloperFlag = cli.BoolFlag{ - Name: "dev", - Usage: "Ephemeral proof-of-authority network with a pre-funded developer account, mining enabled", - } - DeveloperPeriodFlag = cli.IntFlag{ - Name: "dev.period", - Usage: "Block period to use in developer mode (0 = mine only if transaction pending)", - } - DeveloperGasLimitFlag = cli.Uint64Flag{ - Name: "dev.gaslimit", - Usage: "Initial block gas limit", - Value: 11500000, - } - IdentityFlag = cli.StringFlag{ - Name: "identity", - Usage: "Custom node name", - } - DocRootFlag = DirectoryFlag{ - Name: "docroot", - Usage: "Document Root for HTTPClient file scheme", - Value: DirectoryString(HomeDir()), - } - ExitWhenSyncedFlag = cli.BoolFlag{ - Name: "exitwhensynced", - Usage: "Exits after block synchronisation completes", - } - IterativeOutputFlag = cli.BoolTFlag{ + DataDirFlag = &flags.DirectoryFlag{ + Name: "datadir", + Usage: "Data directory for the databases and keystore", + Value: flags.DirectoryString(node.DefaultDataDir()), + Category: flags.EthCategory, + } + RemoteDBFlag = &cli.StringFlag{ + Name: "remotedb", + Usage: "URL for remote database", + Category: flags.LoggingCategory, + } + AncientFlag = &flags.DirectoryFlag{ + Name: "datadir.ancient", + Usage: "Root directory for ancient data (default = inside chaindata)", + Category: flags.EthCategory, + } + MinFreeDiskSpaceFlag = &flags.DirectoryFlag{ + Name: "datadir.minfreedisk", + Usage: "Minimum free disk space in MB, once reached triggers auto shut down (default = --cache.gc converted to MB, 0 = disabled)", + Category: flags.EthCategory, + } + KeyStoreDirFlag = &flags.DirectoryFlag{ + Name: "keystore", + Usage: "Directory for the keystore (default = inside the datadir)", + Category: flags.AccountCategory, + } + USBFlag = &cli.BoolFlag{ + Name: "usb", + Usage: "Enable monitoring and management of USB hardware wallets", + Category: flags.AccountCategory, + } + SmartCardDaemonPathFlag = &cli.StringFlag{ + Name: "pcscdpath", + Usage: "Path to the smartcard daemon (pcscd) socket file", + Value: pcsclite.PCSCDSockName, + Category: flags.AccountCategory, + } + NetworkIdFlag = &cli.Uint64Flag{ + Name: "networkid", + Usage: "Explicitly set network id (integer)(For testnets: use --ropsten, --rinkeby, --goerli instead)", + Value: ethconfig.Defaults.NetworkId, + Category: flags.EthCategory, + } + MainnetFlag = &cli.BoolFlag{ + Name: "mainnet", + Usage: "Ethereum mainnet", + Category: flags.EthCategory, + } + RopstenFlag = &cli.BoolFlag{ + Name: "ropsten", + Usage: "Ropsten network: pre-configured proof-of-stake test network", + Category: flags.EthCategory, + } + RinkebyFlag = &cli.BoolFlag{ + Name: "rinkeby", + Usage: "Rinkeby network: pre-configured proof-of-authority test network", + Category: flags.EthCategory, + } + GoerliFlag = &cli.BoolFlag{ + Name: "goerli", + Usage: "Görli network: pre-configured proof-of-authority test network", + Category: flags.EthCategory, + } + SepoliaFlag = &cli.BoolFlag{ + Name: "sepolia", + Usage: "Sepolia network: pre-configured proof-of-work test network", + Category: flags.EthCategory, + } + KilnFlag = &cli.BoolFlag{ + Name: "kiln", + Usage: "Kiln network: pre-configured proof-of-work to proof-of-stake test network", + Category: flags.EthCategory, + } + + // Dev mode + DeveloperFlag = &cli.BoolFlag{ + Name: "dev", + Usage: "Ephemeral proof-of-authority network with a pre-funded developer account, mining enabled", + Category: flags.DevCategory, + } + DeveloperPeriodFlag = &cli.IntFlag{ + Name: "dev.period", + Usage: "Block period to use in developer mode (0 = mine only if transaction pending)", + Category: flags.DevCategory, + } + DeveloperGasLimitFlag = &cli.Uint64Flag{ + Name: "dev.gaslimit", + Usage: "Initial block gas limit", + Value: 11500000, + Category: flags.DevCategory, + } + + IdentityFlag = &cli.StringFlag{ + Name: "identity", + Usage: "Custom node name", + Category: flags.NetworkingCategory, + } + DocRootFlag = &flags.DirectoryFlag{ + Name: "docroot", + Usage: "Document Root for HTTPClient file scheme", + Value: flags.DirectoryString(flags.HomeDir()), + Category: flags.APICategory, + } + ExitWhenSyncedFlag = &cli.BoolFlag{ + Name: "exitwhensynced", + Usage: "Exits after block synchronisation completes", + Category: flags.EthCategory, + } + + // Dump command options. + IterativeOutputFlag = &cli.BoolFlag{ Name: "iterative", Usage: "Print streaming JSON iteratively, delimited by newlines", + Value: true, } - ExcludeStorageFlag = cli.BoolFlag{ + ExcludeStorageFlag = &cli.BoolFlag{ Name: "nostorage", Usage: "Exclude storage entries (save db lookups)", } - IncludeIncompletesFlag = cli.BoolFlag{ + IncludeIncompletesFlag = &cli.BoolFlag{ Name: "incompletes", Usage: "Include accounts for which we don't have the address (missing preimage)", } - ExcludeCodeFlag = cli.BoolFlag{ + ExcludeCodeFlag = &cli.BoolFlag{ Name: "nocode", Usage: "Exclude contract code (save db lookups)", } - StartKeyFlag = cli.StringFlag{ + StartKeyFlag = &cli.StringFlag{ Name: "start", Usage: "Start position. Either a hash or address", Value: "0x0000000000000000000000000000000000000000000000000000000000000000", } - DumpLimitFlag = cli.Uint64Flag{ + DumpLimitFlag = &cli.Uint64Flag{ Name: "limit", Usage: "Max number of elements (0 = no limit)", Value: 0, } + defaultSyncMode = ethconfig.Defaults.SyncMode - SyncModeFlag = TextMarshalerFlag{ - Name: "syncmode", - Usage: `Blockchain sync mode ("snap", "full" or "light")`, - Value: &defaultSyncMode, - } - GCModeFlag = cli.StringFlag{ - Name: "gcmode", - Usage: `Blockchain garbage collection mode ("full", "archive")`, - Value: "full", - } - SnapshotFlag = cli.BoolTFlag{ - Name: "snapshot", - Usage: `Enables snapshot-database mode (default = enable)`, - } - TxLookupLimitFlag = cli.Uint64Flag{ - Name: "txlookuplimit", - Usage: "Number of recent blocks to maintain transactions index for (default = about one year, 0 = entire chain)", - Value: ethconfig.Defaults.TxLookupLimit, - } - LightKDFFlag = cli.BoolFlag{ - Name: "lightkdf", - Usage: "Reduce key-derivation RAM & CPU usage at some expense of KDF strength", - } - EthRequiredBlocksFlag = cli.StringFlag{ - Name: "eth.requiredblocks", - Usage: "Comma separated block number-to-hash mappings to require for peering (=)", - } - LegacyWhitelistFlag = cli.StringFlag{ - Name: "whitelist", - Usage: "Comma separated block number-to-hash mappings to enforce (=) (deprecated in favor of --eth.requiredblocks)", - } - BloomFilterSizeFlag = cli.Uint64Flag{ - Name: "bloomfilter.size", - Usage: "Megabytes of memory allocated to bloom-filter for pruning", - Value: 2048, - } - OverrideArrowGlacierFlag = cli.Uint64Flag{ - Name: "override.arrowglacier", - Usage: "Manually specify Arrow Glacier fork-block, overriding the bundled setting", - } - OverrideTerminalTotalDifficulty = BigFlag{ - Name: "override.terminaltotaldifficulty", - Usage: "Manually specify TerminalTotalDifficulty, overriding the bundled setting", + SyncModeFlag = &flags.TextMarshalerFlag{ + Name: "syncmode", + Usage: `Blockchain sync mode ("snap", "full" or "light")`, + Value: &defaultSyncMode, + Category: flags.EthCategory, + } + GCModeFlag = &cli.StringFlag{ + Name: "gcmode", + Usage: `Blockchain garbage collection mode ("full", "archive")`, + Value: "full", + Category: flags.EthCategory, + } + SnapshotFlag = &cli.BoolFlag{ + Name: "snapshot", + Usage: `Enables snapshot-database mode (default = enable)`, + Value: true, + Category: flags.EthCategory, + } + TxLookupLimitFlag = &cli.Uint64Flag{ + Name: "txlookuplimit", + Usage: "Number of recent blocks to maintain transactions index for (default = about one year, 0 = entire chain)", + Value: ethconfig.Defaults.TxLookupLimit, + Category: flags.EthCategory, + } + LightKDFFlag = &cli.BoolFlag{ + Name: "lightkdf", + Usage: "Reduce key-derivation RAM & CPU usage at some expense of KDF strength", + Category: flags.AccountCategory, + } + EthRequiredBlocksFlag = &cli.StringFlag{ + Name: "eth.requiredblocks", + Usage: "Comma separated block number-to-hash mappings to require for peering (=)", + Category: flags.EthCategory, + } + LegacyWhitelistFlag = &cli.StringFlag{ + Name: "whitelist", + Usage: "Comma separated block number-to-hash mappings to enforce (=) (deprecated in favor of --eth.requiredblocks)", + Category: flags.DeprecatedCategory, + } + BloomFilterSizeFlag = &cli.Uint64Flag{ + Name: "bloomfilter.size", + Usage: "Megabytes of memory allocated to bloom-filter for pruning", + Value: 2048, + Category: flags.EthCategory, + } + OverrideTerminalTotalDifficulty = &flags.BigFlag{ + Name: "override.terminaltotaldifficulty", + Usage: "Manually specify TerminalTotalDifficulty, overriding the bundled setting", + Category: flags.EthCategory, + } + OverrideTerminalTotalDifficultyPassed = &cli.BoolFlag{ + Name: "override.terminaltotaldifficultypassed", + Usage: "Manually specify TerminalTotalDifficultyPassed, overriding the bundled setting", + Category: flags.EthCategory, } // Light server and client settings - LightServeFlag = cli.IntFlag{ - Name: "light.serve", - Usage: "Maximum percentage of time allowed for serving LES requests (multi-threaded processing allows values over 100)", - Value: ethconfig.Defaults.LightServ, - } - LightIngressFlag = cli.IntFlag{ - Name: "light.ingress", - Usage: "Incoming bandwidth limit for serving light clients (kilobytes/sec, 0 = unlimited)", - Value: ethconfig.Defaults.LightIngress, - } - LightEgressFlag = cli.IntFlag{ - Name: "light.egress", - Usage: "Outgoing bandwidth limit for serving light clients (kilobytes/sec, 0 = unlimited)", - Value: ethconfig.Defaults.LightEgress, - } - LightMaxPeersFlag = cli.IntFlag{ - Name: "light.maxpeers", - Usage: "Maximum number of light clients to serve, or light servers to attach to", - Value: ethconfig.Defaults.LightPeers, - } - UltraLightServersFlag = cli.StringFlag{ - Name: "ulc.servers", - Usage: "List of trusted ultra-light servers", - Value: strings.Join(ethconfig.Defaults.UltraLightServers, ","), - } - UltraLightFractionFlag = cli.IntFlag{ - Name: "ulc.fraction", - Usage: "Minimum % of trusted ultra-light servers required to announce a new head", - Value: ethconfig.Defaults.UltraLightFraction, - } - UltraLightOnlyAnnounceFlag = cli.BoolFlag{ - Name: "ulc.onlyannounce", - Usage: "Ultra light server sends announcements only", - } - LightNoPruneFlag = cli.BoolFlag{ - Name: "light.nopruning", - Usage: "Disable ancient light chain data pruning", - } - LightNoSyncServeFlag = cli.BoolFlag{ - Name: "light.nosyncserve", - Usage: "Enables serving light clients before syncing", + LightServeFlag = &cli.IntFlag{ + Name: "light.serve", + Usage: "Maximum percentage of time allowed for serving LES requests (multi-threaded processing allows values over 100)", + Value: ethconfig.Defaults.LightServ, + Category: flags.LightCategory, + } + LightIngressFlag = &cli.IntFlag{ + Name: "light.ingress", + Usage: "Incoming bandwidth limit for serving light clients (kilobytes/sec, 0 = unlimited)", + Value: ethconfig.Defaults.LightIngress, + Category: flags.LightCategory, + } + LightEgressFlag = &cli.IntFlag{ + Name: "light.egress", + Usage: "Outgoing bandwidth limit for serving light clients (kilobytes/sec, 0 = unlimited)", + Value: ethconfig.Defaults.LightEgress, + Category: flags.LightCategory, + } + LightMaxPeersFlag = &cli.IntFlag{ + Name: "light.maxpeers", + Usage: "Maximum number of light clients to serve, or light servers to attach to", + Value: ethconfig.Defaults.LightPeers, + Category: flags.LightCategory, + } + UltraLightServersFlag = &cli.StringFlag{ + Name: "ulc.servers", + Usage: "List of trusted ultra-light servers", + Value: strings.Join(ethconfig.Defaults.UltraLightServers, ","), + Category: flags.LightCategory, + } + UltraLightFractionFlag = &cli.IntFlag{ + Name: "ulc.fraction", + Usage: "Minimum % of trusted ultra-light servers required to announce a new head", + Value: ethconfig.Defaults.UltraLightFraction, + Category: flags.LightCategory, + } + UltraLightOnlyAnnounceFlag = &cli.BoolFlag{ + Name: "ulc.onlyannounce", + Usage: "Ultra light server sends announcements only", + Category: flags.LightCategory, + } + LightNoPruneFlag = &cli.BoolFlag{ + Name: "light.nopruning", + Usage: "Disable ancient light chain data pruning", + Category: flags.LightCategory, + } + LightNoSyncServeFlag = &cli.BoolFlag{ + Name: "light.nosyncserve", + Usage: "Enables serving light clients before syncing", + Category: flags.LightCategory, } + // Ethash settings - EthashCacheDirFlag = DirectoryFlag{ - Name: "ethash.cachedir", - Usage: "Directory to store the ethash verification caches (default = inside the datadir)", - } - EthashCachesInMemoryFlag = cli.IntFlag{ - Name: "ethash.cachesinmem", - Usage: "Number of recent ethash caches to keep in memory (16MB each)", - Value: ethconfig.Defaults.Ethash.CachesInMem, - } - EthashCachesOnDiskFlag = cli.IntFlag{ - Name: "ethash.cachesondisk", - Usage: "Number of recent ethash caches to keep on disk (16MB each)", - Value: ethconfig.Defaults.Ethash.CachesOnDisk, - } - EthashCachesLockMmapFlag = cli.BoolFlag{ - Name: "ethash.cacheslockmmap", - Usage: "Lock memory maps of recent ethash caches", - } - EthashDatasetDirFlag = DirectoryFlag{ - Name: "ethash.dagdir", - Usage: "Directory to store the ethash mining DAGs", - Value: DirectoryString(ethconfig.Defaults.Ethash.DatasetDir), - } - EthashDatasetsInMemoryFlag = cli.IntFlag{ - Name: "ethash.dagsinmem", - Usage: "Number of recent ethash mining DAGs to keep in memory (1+GB each)", - Value: ethconfig.Defaults.Ethash.DatasetsInMem, - } - EthashDatasetsOnDiskFlag = cli.IntFlag{ - Name: "ethash.dagsondisk", - Usage: "Number of recent ethash mining DAGs to keep on disk (1+GB each)", - Value: ethconfig.Defaults.Ethash.DatasetsOnDisk, - } - EthashDatasetsLockMmapFlag = cli.BoolFlag{ - Name: "ethash.dagslockmmap", - Usage: "Lock memory maps for recent ethash mining DAGs", + EthashCacheDirFlag = &flags.DirectoryFlag{ + Name: "ethash.cachedir", + Usage: "Directory to store the ethash verification caches (default = inside the datadir)", + Category: flags.EthashCategory, + } + EthashCachesInMemoryFlag = &cli.IntFlag{ + Name: "ethash.cachesinmem", + Usage: "Number of recent ethash caches to keep in memory (16MB each)", + Value: ethconfig.Defaults.Ethash.CachesInMem, + Category: flags.EthashCategory, + } + EthashCachesOnDiskFlag = &cli.IntFlag{ + Name: "ethash.cachesondisk", + Usage: "Number of recent ethash caches to keep on disk (16MB each)", + Value: ethconfig.Defaults.Ethash.CachesOnDisk, + Category: flags.EthashCategory, + } + EthashCachesLockMmapFlag = &cli.BoolFlag{ + Name: "ethash.cacheslockmmap", + Usage: "Lock memory maps of recent ethash caches", + Category: flags.EthashCategory, + } + EthashDatasetDirFlag = &flags.DirectoryFlag{ + Name: "ethash.dagdir", + Usage: "Directory to store the ethash mining DAGs", + Value: flags.DirectoryString(ethconfig.Defaults.Ethash.DatasetDir), + Category: flags.EthashCategory, + } + EthashDatasetsInMemoryFlag = &cli.IntFlag{ + Name: "ethash.dagsinmem", + Usage: "Number of recent ethash mining DAGs to keep in memory (1+GB each)", + Value: ethconfig.Defaults.Ethash.DatasetsInMem, + Category: flags.EthashCategory, + } + EthashDatasetsOnDiskFlag = &cli.IntFlag{ + Name: "ethash.dagsondisk", + Usage: "Number of recent ethash mining DAGs to keep on disk (1+GB each)", + Value: ethconfig.Defaults.Ethash.DatasetsOnDisk, + Category: flags.EthashCategory, + } + EthashDatasetsLockMmapFlag = &cli.BoolFlag{ + Name: "ethash.dagslockmmap", + Usage: "Lock memory maps for recent ethash mining DAGs", + Category: flags.EthashCategory, } + // Transaction pool settings - TxPoolLocalsFlag = cli.StringFlag{ - Name: "txpool.locals", - Usage: "Comma separated accounts to treat as locals (no flush, priority inclusion)", - } - TxPoolNoLocalsFlag = cli.BoolFlag{ - Name: "txpool.nolocals", - Usage: "Disables price exemptions for locally submitted transactions", - } - TxPoolJournalFlag = cli.StringFlag{ - Name: "txpool.journal", - Usage: "Disk journal for local transaction to survive node restarts", - Value: core.DefaultTxPoolConfig.Journal, - } - TxPoolRejournalFlag = cli.DurationFlag{ - Name: "txpool.rejournal", - Usage: "Time interval to regenerate the local transaction journal", - Value: core.DefaultTxPoolConfig.Rejournal, - } - TxPoolPriceLimitFlag = cli.Uint64Flag{ - Name: "txpool.pricelimit", - Usage: "Minimum gas price limit to enforce for acceptance into the pool", - Value: ethconfig.Defaults.TxPool.PriceLimit, - } - TxPoolPriceBumpFlag = cli.Uint64Flag{ - Name: "txpool.pricebump", - Usage: "Price bump percentage to replace an already existing transaction", - Value: ethconfig.Defaults.TxPool.PriceBump, - } - TxPoolAccountSlotsFlag = cli.Uint64Flag{ - Name: "txpool.accountslots", - Usage: "Minimum number of executable transaction slots guaranteed per account", - Value: ethconfig.Defaults.TxPool.AccountSlots, - } - TxPoolGlobalSlotsFlag = cli.Uint64Flag{ - Name: "txpool.globalslots", - Usage: "Maximum number of executable transaction slots for all accounts", - Value: ethconfig.Defaults.TxPool.GlobalSlots, - } - TxPoolAccountQueueFlag = cli.Uint64Flag{ - Name: "txpool.accountqueue", - Usage: "Maximum number of non-executable transaction slots permitted per account", - Value: ethconfig.Defaults.TxPool.AccountQueue, - } - TxPoolGlobalQueueFlag = cli.Uint64Flag{ - Name: "txpool.globalqueue", - Usage: "Maximum number of non-executable transaction slots for all accounts", - Value: ethconfig.Defaults.TxPool.GlobalQueue, - } - TxPoolLifetimeFlag = cli.DurationFlag{ - Name: "txpool.lifetime", - Usage: "Maximum amount of time non-executable transaction are queued", - Value: ethconfig.Defaults.TxPool.Lifetime, + TxPoolLocalsFlag = &cli.StringFlag{ + Name: "txpool.locals", + Usage: "Comma separated accounts to treat as locals (no flush, priority inclusion)", + Category: flags.TxPoolCategory, + } + TxPoolNoLocalsFlag = &cli.BoolFlag{ + Name: "txpool.nolocals", + Usage: "Disables price exemptions for locally submitted transactions", + Category: flags.TxPoolCategory, + } + TxPoolJournalFlag = &cli.StringFlag{ + Name: "txpool.journal", + Usage: "Disk journal for local transaction to survive node restarts", + Value: core.DefaultTxPoolConfig.Journal, + Category: flags.TxPoolCategory, + } + TxPoolRejournalFlag = &cli.DurationFlag{ + Name: "txpool.rejournal", + Usage: "Time interval to regenerate the local transaction journal", + Value: core.DefaultTxPoolConfig.Rejournal, + Category: flags.TxPoolCategory, + } + TxPoolPriceLimitFlag = &cli.Uint64Flag{ + Name: "txpool.pricelimit", + Usage: "Minimum gas price limit to enforce for acceptance into the pool", + Value: ethconfig.Defaults.TxPool.PriceLimit, + Category: flags.TxPoolCategory, + } + TxPoolPriceBumpFlag = &cli.Uint64Flag{ + Name: "txpool.pricebump", + Usage: "Price bump percentage to replace an already existing transaction", + Value: ethconfig.Defaults.TxPool.PriceBump, + Category: flags.TxPoolCategory, + } + TxPoolAccountSlotsFlag = &cli.Uint64Flag{ + Name: "txpool.accountslots", + Usage: "Minimum number of executable transaction slots guaranteed per account", + Value: ethconfig.Defaults.TxPool.AccountSlots, + Category: flags.TxPoolCategory, + } + TxPoolGlobalSlotsFlag = &cli.Uint64Flag{ + Name: "txpool.globalslots", + Usage: "Maximum number of executable transaction slots for all accounts", + Value: ethconfig.Defaults.TxPool.GlobalSlots, + Category: flags.TxPoolCategory, + } + TxPoolAccountQueueFlag = &cli.Uint64Flag{ + Name: "txpool.accountqueue", + Usage: "Maximum number of non-executable transaction slots permitted per account", + Value: ethconfig.Defaults.TxPool.AccountQueue, + Category: flags.TxPoolCategory, + } + TxPoolGlobalQueueFlag = &cli.Uint64Flag{ + Name: "txpool.globalqueue", + Usage: "Maximum number of non-executable transaction slots for all accounts", + Value: ethconfig.Defaults.TxPool.GlobalQueue, + Category: flags.TxPoolCategory, + } + TxPoolLifetimeFlag = &cli.DurationFlag{ + Name: "txpool.lifetime", + Usage: "Maximum amount of time non-executable transaction are queued", + Value: ethconfig.Defaults.TxPool.Lifetime, + Category: flags.TxPoolCategory, } + // Performance tuning settings - CacheFlag = cli.IntFlag{ - Name: "cache", - Usage: "Megabytes of memory allocated to internal caching (default = 4096 mainnet full node, 128 light mode)", - Value: 1024, - } - CacheDatabaseFlag = cli.IntFlag{ - Name: "cache.database", - Usage: "Percentage of cache memory allowance to use for database io", - Value: 50, - } - CacheTrieFlag = cli.IntFlag{ - Name: "cache.trie", - Usage: "Percentage of cache memory allowance to use for trie caching (default = 15% full mode, 30% archive mode)", - Value: 15, - } - CacheTrieJournalFlag = cli.StringFlag{ - Name: "cache.trie.journal", - Usage: "Disk journal directory for trie cache to survive node restarts", - Value: ethconfig.Defaults.TrieCleanCacheJournal, - } - CacheTrieRejournalFlag = cli.DurationFlag{ - Name: "cache.trie.rejournal", - Usage: "Time interval to regenerate the trie cache journal", - Value: ethconfig.Defaults.TrieCleanCacheRejournal, - } - CacheGCFlag = cli.IntFlag{ - Name: "cache.gc", - Usage: "Percentage of cache memory allowance to use for trie pruning (default = 25% full mode, 0% archive mode)", - Value: 25, - } - CacheSnapshotFlag = cli.IntFlag{ - Name: "cache.snapshot", - Usage: "Percentage of cache memory allowance to use for snapshot caching (default = 10% full mode, 20% archive mode)", - Value: 10, - } - CacheNoPrefetchFlag = cli.BoolFlag{ - Name: "cache.noprefetch", - Usage: "Disable heuristic state prefetch during block import (less CPU and disk IO, more time waiting for data)", - } - CachePreimagesFlag = cli.BoolFlag{ - Name: "cache.preimages", - Usage: "Enable recording the SHA3/keccak preimages of trie keys", - } - FDLimitFlag = cli.IntFlag{ - Name: "fdlimit", - Usage: "Raise the open file descriptor resource limit (default = system fd limit)", + CacheFlag = &cli.IntFlag{ + Name: "cache", + Usage: "Megabytes of memory allocated to internal caching (default = 4096 mainnet full node, 128 light mode)", + Value: 1024, + Category: flags.PerfCategory, + } + CacheDatabaseFlag = &cli.IntFlag{ + Name: "cache.database", + Usage: "Percentage of cache memory allowance to use for database io", + Value: 50, + Category: flags.PerfCategory, + } + CacheTrieFlag = &cli.IntFlag{ + Name: "cache.trie", + Usage: "Percentage of cache memory allowance to use for trie caching (default = 15% full mode, 30% archive mode)", + Value: 15, + Category: flags.PerfCategory, + } + CacheTrieJournalFlag = &cli.StringFlag{ + Name: "cache.trie.journal", + Usage: "Disk journal directory for trie cache to survive node restarts", + Value: ethconfig.Defaults.TrieCleanCacheJournal, + Category: flags.PerfCategory, + } + CacheTrieRejournalFlag = &cli.DurationFlag{ + Name: "cache.trie.rejournal", + Usage: "Time interval to regenerate the trie cache journal", + Value: ethconfig.Defaults.TrieCleanCacheRejournal, + Category: flags.PerfCategory, + } + CacheGCFlag = &cli.IntFlag{ + Name: "cache.gc", + Usage: "Percentage of cache memory allowance to use for trie pruning (default = 25% full mode, 0% archive mode)", + Value: 25, + Category: flags.PerfCategory, + } + CacheSnapshotFlag = &cli.IntFlag{ + Name: "cache.snapshot", + Usage: "Percentage of cache memory allowance to use for snapshot caching (default = 10% full mode, 20% archive mode)", + Value: 10, + Category: flags.PerfCategory, + } + CacheNoPrefetchFlag = &cli.BoolFlag{ + Name: "cache.noprefetch", + Usage: "Disable heuristic state prefetch during block import (less CPU and disk IO, more time waiting for data)", + Category: flags.PerfCategory, + } + CachePreimagesFlag = &cli.BoolFlag{ + Name: "cache.preimages", + Usage: "Enable recording the SHA3/keccak preimages of trie keys", + Category: flags.PerfCategory, + } + CacheLogSizeFlag = &cli.IntFlag{ + Name: "cache.blocklogs", + Usage: "Size (in number of blocks) of the log cache for filtering", + Category: flags.PerfCategory, + Value: ethconfig.Defaults.FilterLogCacheSize, + } + FDLimitFlag = &cli.IntFlag{ + Name: "fdlimit", + Usage: "Raise the open file descriptor resource limit (default = system fd limit)", + Category: flags.PerfCategory, } + // Miner settings - MiningEnabledFlag = cli.BoolFlag{ - Name: "mine", - Usage: "Enable mining", - } - MinerThreadsFlag = cli.IntFlag{ - Name: "miner.threads", - Usage: "Number of CPU threads to use for mining", - Value: 0, - } - MinerNotifyFlag = cli.StringFlag{ - Name: "miner.notify", - Usage: "Comma separated HTTP URL list to notify of new work packages", - } - MinerNotifyFullFlag = cli.BoolFlag{ - Name: "miner.notify.full", - Usage: "Notify with pending block headers instead of work packages", - } - MinerGasLimitFlag = cli.Uint64Flag{ - Name: "miner.gaslimit", - Usage: "Target gas ceiling for mined blocks", - Value: ethconfig.Defaults.Miner.GasCeil, - } - MinerGasPriceFlag = BigFlag{ - Name: "miner.gasprice", - Usage: "Minimum gas price for mining a transaction", - Value: ethconfig.Defaults.Miner.GasPrice, - } - MinerEtherbaseFlag = cli.StringFlag{ - Name: "miner.etherbase", - Usage: "Public address for block mining rewards (default = first account)", - Value: "0", - } - MinerExtraDataFlag = cli.StringFlag{ - Name: "miner.extradata", - Usage: "Block extra data set by the miner (default = client version)", - } - MinerRecommitIntervalFlag = cli.DurationFlag{ - Name: "miner.recommit", - Usage: "Time interval to recreate the block being mined", - Value: ethconfig.Defaults.Miner.Recommit, - } - MinerNoVerifyFlag = cli.BoolFlag{ - Name: "miner.noverify", - Usage: "Disable remote sealing verification", + MiningEnabledFlag = &cli.BoolFlag{ + Name: "mine", + Usage: "Enable mining", + Category: flags.MinerCategory, + } + MinerThreadsFlag = &cli.IntFlag{ + Name: "miner.threads", + Usage: "Number of CPU threads to use for mining", + Value: 0, + Category: flags.MinerCategory, + } + MinerNotifyFlag = &cli.StringFlag{ + Name: "miner.notify", + Usage: "Comma separated HTTP URL list to notify of new work packages", + Category: flags.MinerCategory, + } + MinerNotifyFullFlag = &cli.BoolFlag{ + Name: "miner.notify.full", + Usage: "Notify with pending block headers instead of work packages", + Category: flags.MinerCategory, + } + MinerGasLimitFlag = &cli.Uint64Flag{ + Name: "miner.gaslimit", + Usage: "Target gas ceiling for mined blocks", + Value: ethconfig.Defaults.Miner.GasCeil, + Category: flags.MinerCategory, + } + MinerGasPriceFlag = &flags.BigFlag{ + Name: "miner.gasprice", + Usage: "Minimum gas price for mining a transaction", + Value: ethconfig.Defaults.Miner.GasPrice, + Category: flags.MinerCategory, + } + MinerEtherbaseFlag = &cli.StringFlag{ + Name: "miner.etherbase", + Usage: "Public address for block mining rewards (default = first account)", + Value: "0", + Category: flags.MinerCategory, + } + MinerExtraDataFlag = &cli.StringFlag{ + Name: "miner.extradata", + Usage: "Block extra data set by the miner (default = client version)", + Category: flags.MinerCategory, + } + MinerRecommitIntervalFlag = &cli.DurationFlag{ + Name: "miner.recommit", + Usage: "Time interval to recreate the block being mined", + Value: ethconfig.Defaults.Miner.Recommit, + Category: flags.MinerCategory, + } + MinerNoVerifyFlag = &cli.BoolFlag{ + Name: "miner.noverify", + Usage: "Disable remote sealing verification", + Category: flags.MinerCategory, } + // Account settings - UnlockedAccountFlag = cli.StringFlag{ - Name: "unlock", - Usage: "Comma separated list of accounts to unlock", - Value: "", - } - PasswordFileFlag = cli.StringFlag{ - Name: "password", - Usage: "Password file to use for non-interactive password input", - Value: "", - } - ExternalSignerFlag = cli.StringFlag{ - Name: "signer", - Usage: "External signer (url or path to ipc file)", - Value: "", - } - VMEnableDebugFlag = cli.BoolFlag{ - Name: "vmdebug", - Usage: "Record information useful for VM and contract debugging", - } - InsecureUnlockAllowedFlag = cli.BoolFlag{ - Name: "allow-insecure-unlock", - Usage: "Allow insecure account unlocking when account-related RPCs are exposed by http", - } - RPCGlobalGasCapFlag = cli.Uint64Flag{ - Name: "rpc.gascap", - Usage: "Sets a cap on gas that can be used in eth_call/estimateGas (0=infinite)", - Value: ethconfig.Defaults.RPCGasCap, - } - RPCGlobalEVMTimeoutFlag = cli.DurationFlag{ - Name: "rpc.evmtimeout", - Usage: "Sets a timeout used for eth_call (0=infinite)", - Value: ethconfig.Defaults.RPCEVMTimeout, - } - RPCGlobalTxFeeCapFlag = cli.Float64Flag{ - Name: "rpc.txfeecap", - Usage: "Sets a cap on transaction fee (in ether) that can be sent via the RPC APIs (0 = no cap)", - Value: ethconfig.Defaults.RPCTxFeeCap, + UnlockedAccountFlag = &cli.StringFlag{ + Name: "unlock", + Usage: "Comma separated list of accounts to unlock", + Value: "", + Category: flags.AccountCategory, + } + PasswordFileFlag = &cli.PathFlag{ + Name: "password", + Usage: "Password file to use for non-interactive password input", + TakesFile: true, + Category: flags.AccountCategory, + } + ExternalSignerFlag = &cli.StringFlag{ + Name: "signer", + Usage: "External signer (url or path to ipc file)", + Value: "", + Category: flags.AccountCategory, + } + InsecureUnlockAllowedFlag = &cli.BoolFlag{ + Name: "allow-insecure-unlock", + Usage: "Allow insecure account unlocking when account-related RPCs are exposed by http", + Category: flags.AccountCategory, + } + + // EVM settings + VMEnableDebugFlag = &cli.BoolFlag{ + Name: "vmdebug", + Usage: "Record information useful for VM and contract debugging", + Category: flags.VMCategory, + } + + // API options. + RPCGlobalGasCapFlag = &cli.Uint64Flag{ + Name: "rpc.gascap", + Usage: "Sets a cap on gas that can be used in eth_call/estimateGas (0=infinite)", + Value: ethconfig.Defaults.RPCGasCap, + Category: flags.APICategory, + } + RPCGlobalEVMTimeoutFlag = &cli.DurationFlag{ + Name: "rpc.evmtimeout", + Usage: "Sets a timeout used for eth_call (0=infinite)", + Value: ethconfig.Defaults.RPCEVMTimeout, + Category: flags.APICategory, + } + RPCGlobalTxFeeCapFlag = &cli.Float64Flag{ + Name: "rpc.txfeecap", + Usage: "Sets a cap on transaction fee (in ether) that can be sent via the RPC APIs (0 = no cap)", + Value: ethconfig.Defaults.RPCTxFeeCap, + Category: flags.APICategory, } // Authenticated RPC HTTP settings - AuthListenFlag = cli.StringFlag{ - Name: "authrpc.addr", - Usage: "Listening address for authenticated APIs", - Value: node.DefaultConfig.AuthAddr, - } - AuthPortFlag = cli.IntFlag{ - Name: "authrpc.port", - Usage: "Listening port for authenticated APIs", - Value: node.DefaultConfig.AuthPort, - } - AuthVirtualHostsFlag = cli.StringFlag{ - Name: "authrpc.vhosts", - Usage: "Comma separated list of virtual hostnames from which to accept requests (server enforced). Accepts '*' wildcard.", - Value: strings.Join(node.DefaultConfig.AuthVirtualHosts, ","), - } - JWTSecretFlag = cli.StringFlag{ - Name: "authrpc.jwtsecret", - Usage: "Path to a JWT secret to use for authenticated RPC endpoints", + AuthListenFlag = &cli.StringFlag{ + Name: "authrpc.addr", + Usage: "Listening address for authenticated APIs", + Value: node.DefaultConfig.AuthAddr, + Category: flags.APICategory, + } + AuthPortFlag = &cli.IntFlag{ + Name: "authrpc.port", + Usage: "Listening port for authenticated APIs", + Value: node.DefaultConfig.AuthPort, + Category: flags.APICategory, + } + AuthVirtualHostsFlag = &cli.StringFlag{ + Name: "authrpc.vhosts", + Usage: "Comma separated list of virtual hostnames from which to accept requests (server enforced). Accepts '*' wildcard.", + Value: strings.Join(node.DefaultConfig.AuthVirtualHosts, ","), + Category: flags.APICategory, + } + JWTSecretFlag = &cli.StringFlag{ + Name: "authrpc.jwtsecret", + Usage: "Path to a JWT secret to use for authenticated RPC endpoints", + Category: flags.APICategory, } + // Logging and debug settings - EthStatsURLFlag = cli.StringFlag{ - Name: "ethstats", - Usage: "Reporting URL of a ethstats service (nodename:secret@host:port)", + EthStatsURLFlag = &cli.StringFlag{ + Name: "ethstats", + Usage: "Reporting URL of a ethstats service (nodename:secret@host:port)", + Category: flags.MetricsCategory, } - FakePoWFlag = cli.BoolFlag{ - Name: "fakepow", - Usage: "Disables proof-of-work verification", + FakePoWFlag = &cli.BoolFlag{ + Name: "fakepow", + Usage: "Disables proof-of-work verification", + Category: flags.LoggingCategory, } - NoCompactionFlag = cli.BoolFlag{ - Name: "nocompaction", - Usage: "Disables db compaction after import", - } - // RPC settings - IPCDisabledFlag = cli.BoolFlag{ - Name: "ipcdisable", - Usage: "Disable the IPC-RPC server", - } - IPCPathFlag = DirectoryFlag{ - Name: "ipcpath", - Usage: "Filename for IPC socket/pipe within the datadir (explicit paths escape it)", - } - HTTPEnabledFlag = cli.BoolFlag{ - Name: "http", - Usage: "Enable the HTTP-RPC server", - } - HTTPListenAddrFlag = cli.StringFlag{ - Name: "http.addr", - Usage: "HTTP-RPC server listening interface", - Value: node.DefaultHTTPHost, - } - HTTPPortFlag = cli.IntFlag{ - Name: "http.port", - Usage: "HTTP-RPC server listening port", - Value: node.DefaultHTTPPort, - } - HTTPCORSDomainFlag = cli.StringFlag{ - Name: "http.corsdomain", - Usage: "Comma separated list of domains from which to accept cross origin requests (browser enforced)", - Value: "", - } - HTTPVirtualHostsFlag = cli.StringFlag{ - Name: "http.vhosts", - Usage: "Comma separated list of virtual hostnames from which to accept requests (server enforced). Accepts '*' wildcard.", - Value: strings.Join(node.DefaultConfig.HTTPVirtualHosts, ","), - } - HTTPApiFlag = cli.StringFlag{ - Name: "http.api", - Usage: "API's offered over the HTTP-RPC interface", - Value: "", - } - HTTPPathPrefixFlag = cli.StringFlag{ - Name: "http.rpcprefix", - Usage: "HTTP path path prefix on which JSON-RPC is served. Use '/' to serve on all paths.", - Value: "", - } - GraphQLEnabledFlag = cli.BoolFlag{ - Name: "graphql", - Usage: "Enable GraphQL on the HTTP-RPC server. Note that GraphQL can only be started if an HTTP server is started as well.", - } - GraphQLCORSDomainFlag = cli.StringFlag{ - Name: "graphql.corsdomain", - Usage: "Comma separated list of domains from which to accept cross origin requests (browser enforced)", - Value: "", - } - GraphQLVirtualHostsFlag = cli.StringFlag{ - Name: "graphql.vhosts", - Usage: "Comma separated list of virtual hostnames from which to accept requests (server enforced). Accepts '*' wildcard.", - Value: strings.Join(node.DefaultConfig.GraphQLVirtualHosts, ","), - } - WSEnabledFlag = cli.BoolFlag{ - Name: "ws", - Usage: "Enable the WS-RPC server", - } - WSListenAddrFlag = cli.StringFlag{ - Name: "ws.addr", - Usage: "WS-RPC server listening interface", - Value: node.DefaultWSHost, - } - WSPortFlag = cli.IntFlag{ - Name: "ws.port", - Usage: "WS-RPC server listening port", - Value: node.DefaultWSPort, - } - WSApiFlag = cli.StringFlag{ - Name: "ws.api", - Usage: "API's offered over the WS-RPC interface", - Value: "", - } - WSAllowedOriginsFlag = cli.StringFlag{ - Name: "ws.origins", - Usage: "Origins from which to accept websockets requests", - Value: "", - } - WSPathPrefixFlag = cli.StringFlag{ - Name: "ws.rpcprefix", - Usage: "HTTP path prefix on which JSON-RPC is served. Use '/' to serve on all paths.", - Value: "", - } - ExecFlag = cli.StringFlag{ - Name: "exec", - Usage: "Execute JavaScript statement", - } - PreloadJSFlag = cli.StringFlag{ - Name: "preload", - Usage: "Comma separated list of JavaScript files to preload into the console", - } - AllowUnprotectedTxs = cli.BoolFlag{ - Name: "rpc.allow-unprotected-txs", - Usage: "Allow for unprotected (non EIP155 signed) transactions to be submitted via RPC", + NoCompactionFlag = &cli.BoolFlag{ + Name: "nocompaction", + Usage: "Disables db compaction after import", + Category: flags.LoggingCategory, } - // Network Settings - MaxPeersFlag = cli.IntFlag{ - Name: "maxpeers", - Usage: "Maximum number of network peers (network disabled if set to 0)", - Value: node.DefaultConfig.P2P.MaxPeers, - } - MaxPendingPeersFlag = cli.IntFlag{ - Name: "maxpendpeers", - Usage: "Maximum number of pending connection attempts (defaults used if set to 0)", - Value: node.DefaultConfig.P2P.MaxPendingPeers, - } - ListenPortFlag = cli.IntFlag{ - Name: "port", - Usage: "Network listening port", - Value: 30303, - } - BootnodesFlag = cli.StringFlag{ - Name: "bootnodes", - Usage: "Comma separated enode URLs for P2P discovery bootstrap", - Value: "", - } - NodeKeyFileFlag = cli.StringFlag{ - Name: "nodekey", - Usage: "P2P node key file", - } - NodeKeyHexFlag = cli.StringFlag{ - Name: "nodekeyhex", - Usage: "P2P node key as hex (for testing)", + IgnoreLegacyReceiptsFlag = &cli.BoolFlag{ + Name: "ignore-legacy-receipts", + Usage: "Geth will start up even if there are legacy receipts in freezer", + Category: flags.MiscCategory, } - NATFlag = cli.StringFlag{ - Name: "nat", - Usage: "NAT port mapping mechanism (any|none|upnp|pmp|extip:)", - Value: "any", - } - NoDiscoverFlag = cli.BoolFlag{ - Name: "nodiscover", - Usage: "Disables the peer discovery mechanism (manual peer addition)", - } - DiscoveryV5Flag = cli.BoolFlag{ - Name: "v5disc", - Usage: "Enables the experimental RLPx V5 (Topic Discovery) mechanism", - } - NetrestrictFlag = cli.StringFlag{ - Name: "netrestrict", - Usage: "Restricts network communication to the given IP networks (CIDR masks)", - } - DNSDiscoveryFlag = cli.StringFlag{ - Name: "discovery.dns", - Usage: "Sets DNS discovery entry points (use \"\" to disable DNS)", + + // RPC settings + IPCDisabledFlag = &cli.BoolFlag{ + Name: "ipcdisable", + Usage: "Disable the IPC-RPC server", + Category: flags.APICategory, + } + IPCPathFlag = &flags.DirectoryFlag{ + Name: "ipcpath", + Usage: "Filename for IPC socket/pipe within the datadir (explicit paths escape it)", + Category: flags.APICategory, + } + HTTPEnabledFlag = &cli.BoolFlag{ + Name: "http", + Usage: "Enable the HTTP-RPC server", + Category: flags.APICategory, + } + HTTPListenAddrFlag = &cli.StringFlag{ + Name: "http.addr", + Usage: "HTTP-RPC server listening interface", + Value: node.DefaultHTTPHost, + Category: flags.APICategory, + } + HTTPPortFlag = &cli.IntFlag{ + Name: "http.port", + Usage: "HTTP-RPC server listening port", + Value: node.DefaultHTTPPort, + Category: flags.APICategory, + } + HTTPCORSDomainFlag = &cli.StringFlag{ + Name: "http.corsdomain", + Usage: "Comma separated list of domains from which to accept cross origin requests (browser enforced)", + Value: "", + Category: flags.APICategory, + } + HTTPVirtualHostsFlag = &cli.StringFlag{ + Name: "http.vhosts", + Usage: "Comma separated list of virtual hostnames from which to accept requests (server enforced). Accepts '*' wildcard.", + Value: strings.Join(node.DefaultConfig.HTTPVirtualHosts, ","), + Category: flags.APICategory, + } + HTTPApiFlag = &cli.StringFlag{ + Name: "http.api", + Usage: "API's offered over the HTTP-RPC interface", + Value: "", + Category: flags.APICategory, + } + HTTPPathPrefixFlag = &cli.StringFlag{ + Name: "http.rpcprefix", + Usage: "HTTP path path prefix on which JSON-RPC is served. Use '/' to serve on all paths.", + Value: "", + Category: flags.APICategory, + } + GraphQLEnabledFlag = &cli.BoolFlag{ + Name: "graphql", + Usage: "Enable GraphQL on the HTTP-RPC server. Note that GraphQL can only be started if an HTTP server is started as well.", + Category: flags.APICategory, + } + GraphQLCORSDomainFlag = &cli.StringFlag{ + Name: "graphql.corsdomain", + Usage: "Comma separated list of domains from which to accept cross origin requests (browser enforced)", + Value: "", + Category: flags.APICategory, + } + GraphQLVirtualHostsFlag = &cli.StringFlag{ + Name: "graphql.vhosts", + Usage: "Comma separated list of virtual hostnames from which to accept requests (server enforced). Accepts '*' wildcard.", + Value: strings.Join(node.DefaultConfig.GraphQLVirtualHosts, ","), + Category: flags.APICategory, + } + WSEnabledFlag = &cli.BoolFlag{ + Name: "ws", + Usage: "Enable the WS-RPC server", + Category: flags.APICategory, + } + WSListenAddrFlag = &cli.StringFlag{ + Name: "ws.addr", + Usage: "WS-RPC server listening interface", + Value: node.DefaultWSHost, + Category: flags.APICategory, + } + WSPortFlag = &cli.IntFlag{ + Name: "ws.port", + Usage: "WS-RPC server listening port", + Value: node.DefaultWSPort, + Category: flags.APICategory, + } + WSApiFlag = &cli.StringFlag{ + Name: "ws.api", + Usage: "API's offered over the WS-RPC interface", + Value: "", + Category: flags.APICategory, + } + WSAllowedOriginsFlag = &cli.StringFlag{ + Name: "ws.origins", + Usage: "Origins from which to accept websockets requests", + Value: "", + Category: flags.APICategory, + } + WSPathPrefixFlag = &cli.StringFlag{ + Name: "ws.rpcprefix", + Usage: "HTTP path prefix on which JSON-RPC is served. Use '/' to serve on all paths.", + Value: "", + Category: flags.APICategory, + } + ExecFlag = &cli.StringFlag{ + Name: "exec", + Usage: "Execute JavaScript statement", + Category: flags.APICategory, + } + PreloadJSFlag = &cli.StringFlag{ + Name: "preload", + Usage: "Comma separated list of JavaScript files to preload into the console", + Category: flags.APICategory, + } + AllowUnprotectedTxs = &cli.BoolFlag{ + Name: "rpc.allow-unprotected-txs", + Usage: "Allow for unprotected (non EIP155 signed) transactions to be submitted via RPC", + Category: flags.APICategory, } - // ATM the url is left to the user and deployment to - JSpathFlag = DirectoryFlag{ - Name: "jspath", - Usage: "JavaScript root path for `loadScript`", - Value: DirectoryString("."), + // Network Settings + MaxPeersFlag = &cli.IntFlag{ + Name: "maxpeers", + Usage: "Maximum number of network peers (network disabled if set to 0)", + Value: node.DefaultConfig.P2P.MaxPeers, + Category: flags.NetworkingCategory, + } + MaxPendingPeersFlag = &cli.IntFlag{ + Name: "maxpendpeers", + Usage: "Maximum number of pending connection attempts (defaults used if set to 0)", + Value: node.DefaultConfig.P2P.MaxPendingPeers, + Category: flags.NetworkingCategory, + } + ListenPortFlag = &cli.IntFlag{ + Name: "port", + Usage: "Network listening port", + Value: 30303, + Category: flags.NetworkingCategory, + } + BootnodesFlag = &cli.StringFlag{ + Name: "bootnodes", + Usage: "Comma separated enode URLs for P2P discovery bootstrap", + Value: "", + Category: flags.NetworkingCategory, + } + NodeKeyFileFlag = &cli.StringFlag{ + Name: "nodekey", + Usage: "P2P node key file", + Category: flags.NetworkingCategory, + } + NodeKeyHexFlag = &cli.StringFlag{ + Name: "nodekeyhex", + Usage: "P2P node key as hex (for testing)", + Category: flags.NetworkingCategory, + } + NATFlag = &cli.StringFlag{ + Name: "nat", + Usage: "NAT port mapping mechanism (any|none|upnp|pmp|extip:)", + Value: "any", + Category: flags.NetworkingCategory, + } + NoDiscoverFlag = &cli.BoolFlag{ + Name: "nodiscover", + Usage: "Disables the peer discovery mechanism (manual peer addition)", + Category: flags.NetworkingCategory, + } + DiscoveryV5Flag = &cli.BoolFlag{ + Name: "v5disc", + Usage: "Enables the experimental RLPx V5 (Topic Discovery) mechanism", + Category: flags.NetworkingCategory, + } + NetrestrictFlag = &cli.StringFlag{ + Name: "netrestrict", + Usage: "Restricts network communication to the given IP networks (CIDR masks)", + Category: flags.NetworkingCategory, + } + DNSDiscoveryFlag = &cli.StringFlag{ + Name: "discovery.dns", + Usage: "Sets DNS discovery entry points (use \"\" to disable DNS)", + Category: flags.NetworkingCategory, + } + DiscoveryPortFlag = &cli.IntFlag{ + Name: "discovery.port", + Usage: "Use a custom UDP port for P2P discovery", + Value: 30303, + Category: flags.NetworkingCategory, + } + + // Console + JSpathFlag = &flags.DirectoryFlag{ + Name: "jspath", + Usage: "JavaScript root path for `loadScript`", + Value: flags.DirectoryString("."), + Category: flags.APICategory, } // Gas price oracle settings - GpoBlocksFlag = cli.IntFlag{ - Name: "gpo.blocks", - Usage: "Number of recent blocks to check for gas prices", - Value: ethconfig.Defaults.GPO.Blocks, - } - GpoPercentileFlag = cli.IntFlag{ - Name: "gpo.percentile", - Usage: "Suggested gas price is the given percentile of a set of recent transaction gas prices", - Value: ethconfig.Defaults.GPO.Percentile, - } - GpoMaxGasPriceFlag = cli.Int64Flag{ - Name: "gpo.maxprice", - Usage: "Maximum transaction priority fee (or gasprice before London fork) to be recommended by gpo", - Value: ethconfig.Defaults.GPO.MaxPrice.Int64(), - } - GpoIgnoreGasPriceFlag = cli.Int64Flag{ - Name: "gpo.ignoreprice", - Usage: "Gas price below which gpo will ignore transactions", - Value: ethconfig.Defaults.GPO.IgnorePrice.Int64(), + GpoBlocksFlag = &cli.IntFlag{ + Name: "gpo.blocks", + Usage: "Number of recent blocks to check for gas prices", + Value: ethconfig.Defaults.GPO.Blocks, + Category: flags.GasPriceCategory, + } + GpoPercentileFlag = &cli.IntFlag{ + Name: "gpo.percentile", + Usage: "Suggested gas price is the given percentile of a set of recent transaction gas prices", + Value: ethconfig.Defaults.GPO.Percentile, + Category: flags.GasPriceCategory, + } + GpoMaxGasPriceFlag = &cli.Int64Flag{ + Name: "gpo.maxprice", + Usage: "Maximum transaction priority fee (or gasprice before London fork) to be recommended by gpo", + Value: ethconfig.Defaults.GPO.MaxPrice.Int64(), + Category: flags.GasPriceCategory, + } + GpoIgnoreGasPriceFlag = &cli.Int64Flag{ + Name: "gpo.ignoreprice", + Usage: "Gas price below which gpo will ignore transactions", + Value: ethconfig.Defaults.GPO.IgnorePrice.Int64(), + Category: flags.GasPriceCategory, } // Metrics flags - MetricsEnabledFlag = cli.BoolFlag{ - Name: "metrics", - Usage: "Enable metrics collection and reporting", + MetricsEnabledFlag = &cli.BoolFlag{ + Name: "metrics", + Usage: "Enable metrics collection and reporting", + Category: flags.MetricsCategory, } - MetricsEnabledExpensiveFlag = cli.BoolFlag{ - Name: "metrics.expensive", - Usage: "Enable expensive metrics collection and reporting", + MetricsEnabledExpensiveFlag = &cli.BoolFlag{ + Name: "metrics.expensive", + Usage: "Enable expensive metrics collection and reporting", + Category: flags.MetricsCategory, } // MetricsHTTPFlag defines the endpoint for a stand-alone metrics HTTP endpoint. // Since the pprof service enables sensitive/vulnerable behavior, this allows a user // to enable a public-OK metrics endpoint without having to worry about ALSO exposing // other profiling behavior or information. - MetricsHTTPFlag = cli.StringFlag{ - Name: "metrics.addr", - Usage: "Enable stand-alone metrics HTTP server listening interface", - Value: metrics.DefaultConfig.HTTP, - } - MetricsPortFlag = cli.IntFlag{ - Name: "metrics.port", - Usage: "Metrics HTTP server listening port", - Value: metrics.DefaultConfig.Port, - } - MetricsEnableInfluxDBFlag = cli.BoolFlag{ - Name: "metrics.influxdb", - Usage: "Enable metrics export/push to an external InfluxDB database", - } - MetricsInfluxDBEndpointFlag = cli.StringFlag{ - Name: "metrics.influxdb.endpoint", - Usage: "InfluxDB API endpoint to report metrics to", - Value: metrics.DefaultConfig.InfluxDBEndpoint, - } - MetricsInfluxDBDatabaseFlag = cli.StringFlag{ - Name: "metrics.influxdb.database", - Usage: "InfluxDB database name to push reported metrics to", - Value: metrics.DefaultConfig.InfluxDBDatabase, - } - MetricsInfluxDBUsernameFlag = cli.StringFlag{ - Name: "metrics.influxdb.username", - Usage: "Username to authorize access to the database", - Value: metrics.DefaultConfig.InfluxDBUsername, - } - MetricsInfluxDBPasswordFlag = cli.StringFlag{ - Name: "metrics.influxdb.password", - Usage: "Password to authorize access to the database", - Value: metrics.DefaultConfig.InfluxDBPassword, + MetricsHTTPFlag = &cli.StringFlag{ + Name: "metrics.addr", + Usage: "Enable stand-alone metrics HTTP server listening interface", + Value: metrics.DefaultConfig.HTTP, + Category: flags.MetricsCategory, + } + MetricsPortFlag = &cli.IntFlag{ + Name: "metrics.port", + Usage: "Metrics HTTP server listening port", + Value: metrics.DefaultConfig.Port, + Category: flags.MetricsCategory, + } + MetricsEnableInfluxDBFlag = &cli.BoolFlag{ + Name: "metrics.influxdb", + Usage: "Enable metrics export/push to an external InfluxDB database", + Category: flags.MetricsCategory, + } + MetricsInfluxDBEndpointFlag = &cli.StringFlag{ + Name: "metrics.influxdb.endpoint", + Usage: "InfluxDB API endpoint to report metrics to", + Value: metrics.DefaultConfig.InfluxDBEndpoint, + Category: flags.MetricsCategory, + } + MetricsInfluxDBDatabaseFlag = &cli.StringFlag{ + Name: "metrics.influxdb.database", + Usage: "InfluxDB database name to push reported metrics to", + Value: metrics.DefaultConfig.InfluxDBDatabase, + Category: flags.MetricsCategory, + } + MetricsInfluxDBUsernameFlag = &cli.StringFlag{ + Name: "metrics.influxdb.username", + Usage: "Username to authorize access to the database", + Value: metrics.DefaultConfig.InfluxDBUsername, + Category: flags.MetricsCategory, + } + MetricsInfluxDBPasswordFlag = &cli.StringFlag{ + Name: "metrics.influxdb.password", + Usage: "Password to authorize access to the database", + Value: metrics.DefaultConfig.InfluxDBPassword, + Category: flags.MetricsCategory, } // Tags are part of every measurement sent to InfluxDB. Queries on tags are faster in InfluxDB. // For example `host` tag could be used so that we can group all nodes and average a measurement // across all of them, but also so that we can select a specific node and inspect its measurements. // https://docs.influxdata.com/influxdb/v1.4/concepts/key_concepts/#tag-key - MetricsInfluxDBTagsFlag = cli.StringFlag{ - Name: "metrics.influxdb.tags", - Usage: "Comma-separated InfluxDB tags (key/values) attached to all measurements", - Value: metrics.DefaultConfig.InfluxDBTags, + MetricsInfluxDBTagsFlag = &cli.StringFlag{ + Name: "metrics.influxdb.tags", + Usage: "Comma-separated InfluxDB tags (key/values) attached to all measurements", + Value: metrics.DefaultConfig.InfluxDBTags, + Category: flags.MetricsCategory, } - MetricsEnableInfluxDBV2Flag = cli.BoolFlag{ - Name: "metrics.influxdbv2", - Usage: "Enable metrics export/push to an external InfluxDB v2 database", + MetricsEnableInfluxDBV2Flag = &cli.BoolFlag{ + Name: "metrics.influxdbv2", + Usage: "Enable metrics export/push to an external InfluxDB v2 database", + Category: flags.MetricsCategory, } - MetricsInfluxDBTokenFlag = cli.StringFlag{ - Name: "metrics.influxdb.token", - Usage: "Token to authorize access to the database (v2 only)", - Value: metrics.DefaultConfig.InfluxDBToken, + MetricsInfluxDBTokenFlag = &cli.StringFlag{ + Name: "metrics.influxdb.token", + Usage: "Token to authorize access to the database (v2 only)", + Value: metrics.DefaultConfig.InfluxDBToken, + Category: flags.MetricsCategory, } - MetricsInfluxDBBucketFlag = cli.StringFlag{ - Name: "metrics.influxdb.bucket", - Usage: "InfluxDB bucket name to push reported metrics to (v2 only)", - Value: metrics.DefaultConfig.InfluxDBBucket, + MetricsInfluxDBBucketFlag = &cli.StringFlag{ + Name: "metrics.influxdb.bucket", + Usage: "InfluxDB bucket name to push reported metrics to (v2 only)", + Value: metrics.DefaultConfig.InfluxDBBucket, + Category: flags.MetricsCategory, } - MetricsInfluxDBOrganizationFlag = cli.StringFlag{ - Name: "metrics.influxdb.organization", - Usage: "InfluxDB organization name (v2 only)", - Value: metrics.DefaultConfig.InfluxDBOrganization, + MetricsInfluxDBOrganizationFlag = &cli.StringFlag{ + Name: "metrics.influxdb.organization", + Usage: "InfluxDB organization name (v2 only)", + Value: metrics.DefaultConfig.InfluxDBOrganization, + Category: flags.MetricsCategory, } ) @@ -849,35 +1001,26 @@ var ( } ) -// GroupFlags combines the given flag slices together and returns the merged one. -func GroupFlags(groups ...[]cli.Flag) []cli.Flag { - var ret []cli.Flag - for _, group := range groups { - ret = append(ret, group...) - } - return ret -} - // MakeDataDir retrieves the currently requested data directory, terminating // if none (or the empty string) is specified. If the node is starting a testnet, // then a subdirectory of the specified datadir will be used. func MakeDataDir(ctx *cli.Context) string { - if path := ctx.GlobalString(DataDirFlag.Name); path != "" { - if ctx.GlobalBool(RopstenFlag.Name) { + if path := ctx.String(DataDirFlag.Name); path != "" { + if ctx.Bool(RopstenFlag.Name) { // Maintain compatibility with older Geth configurations storing the // Ropsten database in `testnet` instead of `ropsten`. return filepath.Join(path, "ropsten") } - if ctx.GlobalBool(RinkebyFlag.Name) { + if ctx.Bool(RinkebyFlag.Name) { return filepath.Join(path, "rinkeby") } - if ctx.GlobalBool(GoerliFlag.Name) { + if ctx.Bool(GoerliFlag.Name) { return filepath.Join(path, "goerli") } - if ctx.GlobalBool(SepoliaFlag.Name) { + if ctx.Bool(SepoliaFlag.Name) { return filepath.Join(path, "sepolia") } - if ctx.GlobalBool(KilnFlag.Name) { + if ctx.Bool(KilnFlag.Name) { return filepath.Join(path, "kiln") } return path @@ -891,8 +1034,8 @@ func MakeDataDir(ctx *cli.Context) string { // method returns nil and an emphemeral key is to be generated. func setNodeKey(ctx *cli.Context, cfg *p2p.Config) { var ( - hex = ctx.GlobalString(NodeKeyHexFlag.Name) - file = ctx.GlobalString(NodeKeyFileFlag.Name) + hex = ctx.String(NodeKeyHexFlag.Name) + file = ctx.String(NodeKeyFileFlag.Name) key *ecdsa.PrivateKey err error ) @@ -914,7 +1057,7 @@ func setNodeKey(ctx *cli.Context, cfg *p2p.Config) { // setNodeUserIdent creates the user identifier from CLI flags. func setNodeUserIdent(ctx *cli.Context, cfg *node.Config) { - if identity := ctx.GlobalString(IdentityFlag.Name); len(identity) > 0 { + if identity := ctx.String(IdentityFlag.Name); len(identity) > 0 { cfg.UserIdent = identity } } @@ -924,20 +1067,23 @@ func setNodeUserIdent(ctx *cli.Context, cfg *node.Config) { func setBootstrapNodes(ctx *cli.Context, cfg *p2p.Config) { urls := params.MainnetBootnodes switch { - case ctx.GlobalIsSet(BootnodesFlag.Name): - urls = SplitAndTrim(ctx.GlobalString(BootnodesFlag.Name)) - case ctx.GlobalBool(RopstenFlag.Name): + case ctx.IsSet(BootnodesFlag.Name): + urls = SplitAndTrim(ctx.String(BootnodesFlag.Name)) + case ctx.Bool(RopstenFlag.Name): urls = params.RopstenBootnodes - case ctx.GlobalBool(SepoliaFlag.Name): + case ctx.Bool(SepoliaFlag.Name): urls = params.SepoliaBootnodes - case ctx.GlobalBool(RinkebyFlag.Name): + case ctx.Bool(RinkebyFlag.Name): urls = params.RinkebyBootnodes - case ctx.GlobalBool(GoerliFlag.Name): + case ctx.Bool(GoerliFlag.Name): urls = params.GoerliBootnodes - case ctx.GlobalBool(KilnFlag.Name): + case ctx.Bool(KilnFlag.Name): urls = params.KilnBootnodes - case cfg.BootstrapNodes != nil: - return // already set, don't apply defaults. + } + + // don't apply defaults if BootstrapNodes is already set + if cfg.BootstrapNodes != nil { + return } cfg.BootstrapNodes = make([]*enode.Node, 0, len(urls)) @@ -958,8 +1104,8 @@ func setBootstrapNodes(ctx *cli.Context, cfg *p2p.Config) { func setBootstrapNodesV5(ctx *cli.Context, cfg *p2p.Config) { urls := params.V5Bootnodes switch { - case ctx.GlobalIsSet(BootnodesFlag.Name): - urls = SplitAndTrim(ctx.GlobalString(BootnodesFlag.Name)) + case ctx.IsSet(BootnodesFlag.Name): + urls = SplitAndTrim(ctx.String(BootnodesFlag.Name)) case cfg.BootstrapNodesV5 != nil: return // already set, don't apply defaults. } @@ -977,18 +1123,21 @@ func setBootstrapNodesV5(ctx *cli.Context, cfg *p2p.Config) { } } -// setListenAddress creates a TCP listening address string from set command -// line flags. +// setListenAddress creates TCP/UDP listening address strings from set command +// line flags func setListenAddress(ctx *cli.Context, cfg *p2p.Config) { - if ctx.GlobalIsSet(ListenPortFlag.Name) { - cfg.ListenAddr = fmt.Sprintf(":%d", ctx.GlobalInt(ListenPortFlag.Name)) + if ctx.IsSet(ListenPortFlag.Name) { + cfg.ListenAddr = fmt.Sprintf(":%d", ctx.Int(ListenPortFlag.Name)) + } + if ctx.IsSet(DiscoveryPortFlag.Name) { + cfg.DiscAddr = fmt.Sprintf(":%d", ctx.Int(DiscoveryPortFlag.Name)) } } // setNAT creates a port mapper from command line flags. func setNAT(ctx *cli.Context, cfg *p2p.Config) { - if ctx.GlobalIsSet(NATFlag.Name) { - natif, err := nat.Parse(ctx.GlobalString(NATFlag.Name)) + if ctx.IsSet(NATFlag.Name) { + natif, err := nat.Parse(ctx.String(NATFlag.Name)) if err != nil { Fatalf("Option %s: %v", NATFlag.Name, err) } @@ -1011,83 +1160,83 @@ func SplitAndTrim(input string) (ret []string) { // setHTTP creates the HTTP RPC listener interface string from the set // command line flags, returning empty if the HTTP endpoint is disabled. func setHTTP(ctx *cli.Context, cfg *node.Config) { - if ctx.GlobalBool(HTTPEnabledFlag.Name) && cfg.HTTPHost == "" { + if ctx.Bool(HTTPEnabledFlag.Name) && cfg.HTTPHost == "" { cfg.HTTPHost = "127.0.0.1" - if ctx.GlobalIsSet(HTTPListenAddrFlag.Name) { - cfg.HTTPHost = ctx.GlobalString(HTTPListenAddrFlag.Name) + if ctx.IsSet(HTTPListenAddrFlag.Name) { + cfg.HTTPHost = ctx.String(HTTPListenAddrFlag.Name) } } - if ctx.GlobalIsSet(HTTPPortFlag.Name) { - cfg.HTTPPort = ctx.GlobalInt(HTTPPortFlag.Name) + if ctx.IsSet(HTTPPortFlag.Name) { + cfg.HTTPPort = ctx.Int(HTTPPortFlag.Name) } - if ctx.GlobalIsSet(AuthListenFlag.Name) { - cfg.AuthAddr = ctx.GlobalString(AuthListenFlag.Name) + if ctx.IsSet(AuthListenFlag.Name) { + cfg.AuthAddr = ctx.String(AuthListenFlag.Name) } - if ctx.GlobalIsSet(AuthPortFlag.Name) { - cfg.AuthPort = ctx.GlobalInt(AuthPortFlag.Name) + if ctx.IsSet(AuthPortFlag.Name) { + cfg.AuthPort = ctx.Int(AuthPortFlag.Name) } - if ctx.GlobalIsSet(AuthVirtualHostsFlag.Name) { - cfg.AuthVirtualHosts = SplitAndTrim(ctx.GlobalString(AuthVirtualHostsFlag.Name)) + if ctx.IsSet(AuthVirtualHostsFlag.Name) { + cfg.AuthVirtualHosts = SplitAndTrim(ctx.String(AuthVirtualHostsFlag.Name)) } - if ctx.GlobalIsSet(HTTPCORSDomainFlag.Name) { - cfg.HTTPCors = SplitAndTrim(ctx.GlobalString(HTTPCORSDomainFlag.Name)) + if ctx.IsSet(HTTPCORSDomainFlag.Name) { + cfg.HTTPCors = SplitAndTrim(ctx.String(HTTPCORSDomainFlag.Name)) } - if ctx.GlobalIsSet(HTTPApiFlag.Name) { - cfg.HTTPModules = SplitAndTrim(ctx.GlobalString(HTTPApiFlag.Name)) + if ctx.IsSet(HTTPApiFlag.Name) { + cfg.HTTPModules = SplitAndTrim(ctx.String(HTTPApiFlag.Name)) } - if ctx.GlobalIsSet(HTTPVirtualHostsFlag.Name) { - cfg.HTTPVirtualHosts = SplitAndTrim(ctx.GlobalString(HTTPVirtualHostsFlag.Name)) + if ctx.IsSet(HTTPVirtualHostsFlag.Name) { + cfg.HTTPVirtualHosts = SplitAndTrim(ctx.String(HTTPVirtualHostsFlag.Name)) } - if ctx.GlobalIsSet(HTTPPathPrefixFlag.Name) { - cfg.HTTPPathPrefix = ctx.GlobalString(HTTPPathPrefixFlag.Name) + if ctx.IsSet(HTTPPathPrefixFlag.Name) { + cfg.HTTPPathPrefix = ctx.String(HTTPPathPrefixFlag.Name) } - if ctx.GlobalIsSet(AllowUnprotectedTxs.Name) { - cfg.AllowUnprotectedTxs = ctx.GlobalBool(AllowUnprotectedTxs.Name) + if ctx.IsSet(AllowUnprotectedTxs.Name) { + cfg.AllowUnprotectedTxs = ctx.Bool(AllowUnprotectedTxs.Name) } } // setGraphQL creates the GraphQL listener interface string from the set // command line flags, returning empty if the GraphQL endpoint is disabled. func setGraphQL(ctx *cli.Context, cfg *node.Config) { - if ctx.GlobalIsSet(GraphQLCORSDomainFlag.Name) { - cfg.GraphQLCors = SplitAndTrim(ctx.GlobalString(GraphQLCORSDomainFlag.Name)) + if ctx.IsSet(GraphQLCORSDomainFlag.Name) { + cfg.GraphQLCors = SplitAndTrim(ctx.String(GraphQLCORSDomainFlag.Name)) } - if ctx.GlobalIsSet(GraphQLVirtualHostsFlag.Name) { - cfg.GraphQLVirtualHosts = SplitAndTrim(ctx.GlobalString(GraphQLVirtualHostsFlag.Name)) + if ctx.IsSet(GraphQLVirtualHostsFlag.Name) { + cfg.GraphQLVirtualHosts = SplitAndTrim(ctx.String(GraphQLVirtualHostsFlag.Name)) } } // setWS creates the WebSocket RPC listener interface string from the set // command line flags, returning empty if the HTTP endpoint is disabled. func setWS(ctx *cli.Context, cfg *node.Config) { - if ctx.GlobalBool(WSEnabledFlag.Name) && cfg.WSHost == "" { + if ctx.Bool(WSEnabledFlag.Name) && cfg.WSHost == "" { cfg.WSHost = "127.0.0.1" - if ctx.GlobalIsSet(WSListenAddrFlag.Name) { - cfg.WSHost = ctx.GlobalString(WSListenAddrFlag.Name) + if ctx.IsSet(WSListenAddrFlag.Name) { + cfg.WSHost = ctx.String(WSListenAddrFlag.Name) } } - if ctx.GlobalIsSet(WSPortFlag.Name) { - cfg.WSPort = ctx.GlobalInt(WSPortFlag.Name) + if ctx.IsSet(WSPortFlag.Name) { + cfg.WSPort = ctx.Int(WSPortFlag.Name) } - if ctx.GlobalIsSet(WSAllowedOriginsFlag.Name) { - cfg.WSOrigins = SplitAndTrim(ctx.GlobalString(WSAllowedOriginsFlag.Name)) + if ctx.IsSet(WSAllowedOriginsFlag.Name) { + cfg.WSOrigins = SplitAndTrim(ctx.String(WSAllowedOriginsFlag.Name)) } - if ctx.GlobalIsSet(WSApiFlag.Name) { - cfg.WSModules = SplitAndTrim(ctx.GlobalString(WSApiFlag.Name)) + if ctx.IsSet(WSApiFlag.Name) { + cfg.WSModules = SplitAndTrim(ctx.String(WSApiFlag.Name)) } - if ctx.GlobalIsSet(WSPathPrefixFlag.Name) { - cfg.WSPathPrefix = ctx.GlobalString(WSPathPrefixFlag.Name) + if ctx.IsSet(WSPathPrefixFlag.Name) { + cfg.WSPathPrefix = ctx.String(WSPathPrefixFlag.Name) } } @@ -1096,45 +1245,45 @@ func setWS(ctx *cli.Context, cfg *node.Config) { func setIPC(ctx *cli.Context, cfg *node.Config) { CheckExclusive(ctx, IPCDisabledFlag, IPCPathFlag) switch { - case ctx.GlobalBool(IPCDisabledFlag.Name): + case ctx.Bool(IPCDisabledFlag.Name): cfg.IPCPath = "" - case ctx.GlobalIsSet(IPCPathFlag.Name): - cfg.IPCPath = ctx.GlobalString(IPCPathFlag.Name) + case ctx.IsSet(IPCPathFlag.Name): + cfg.IPCPath = ctx.String(IPCPathFlag.Name) } } // setLes configures the les server and ultra light client settings from the command line flags. func setLes(ctx *cli.Context, cfg *ethconfig.Config) { - if ctx.GlobalIsSet(LightServeFlag.Name) { - cfg.LightServ = ctx.GlobalInt(LightServeFlag.Name) + if ctx.IsSet(LightServeFlag.Name) { + cfg.LightServ = ctx.Int(LightServeFlag.Name) } - if ctx.GlobalIsSet(LightIngressFlag.Name) { - cfg.LightIngress = ctx.GlobalInt(LightIngressFlag.Name) + if ctx.IsSet(LightIngressFlag.Name) { + cfg.LightIngress = ctx.Int(LightIngressFlag.Name) } - if ctx.GlobalIsSet(LightEgressFlag.Name) { - cfg.LightEgress = ctx.GlobalInt(LightEgressFlag.Name) + if ctx.IsSet(LightEgressFlag.Name) { + cfg.LightEgress = ctx.Int(LightEgressFlag.Name) } - if ctx.GlobalIsSet(LightMaxPeersFlag.Name) { - cfg.LightPeers = ctx.GlobalInt(LightMaxPeersFlag.Name) + if ctx.IsSet(LightMaxPeersFlag.Name) { + cfg.LightPeers = ctx.Int(LightMaxPeersFlag.Name) } - if ctx.GlobalIsSet(UltraLightServersFlag.Name) { - cfg.UltraLightServers = strings.Split(ctx.GlobalString(UltraLightServersFlag.Name), ",") + if ctx.IsSet(UltraLightServersFlag.Name) { + cfg.UltraLightServers = strings.Split(ctx.String(UltraLightServersFlag.Name), ",") } - if ctx.GlobalIsSet(UltraLightFractionFlag.Name) { - cfg.UltraLightFraction = ctx.GlobalInt(UltraLightFractionFlag.Name) + if ctx.IsSet(UltraLightFractionFlag.Name) { + cfg.UltraLightFraction = ctx.Int(UltraLightFractionFlag.Name) } if cfg.UltraLightFraction <= 0 && cfg.UltraLightFraction > 100 { log.Error("Ultra light fraction is invalid", "had", cfg.UltraLightFraction, "updated", ethconfig.Defaults.UltraLightFraction) cfg.UltraLightFraction = ethconfig.Defaults.UltraLightFraction } - if ctx.GlobalIsSet(UltraLightOnlyAnnounceFlag.Name) { - cfg.UltraLightOnlyAnnounce = ctx.GlobalBool(UltraLightOnlyAnnounceFlag.Name) + if ctx.IsSet(UltraLightOnlyAnnounceFlag.Name) { + cfg.UltraLightOnlyAnnounce = ctx.Bool(UltraLightOnlyAnnounceFlag.Name) } - if ctx.GlobalIsSet(LightNoPruneFlag.Name) { - cfg.LightNoPrune = ctx.GlobalBool(LightNoPruneFlag.Name) + if ctx.IsSet(LightNoPruneFlag.Name) { + cfg.LightNoPrune = ctx.Bool(LightNoPruneFlag.Name) } - if ctx.GlobalIsSet(LightNoSyncServeFlag.Name) { - cfg.LightNoSyncServe = ctx.GlobalBool(LightNoSyncServeFlag.Name) + if ctx.IsSet(LightNoSyncServeFlag.Name) { + cfg.LightNoSyncServe = ctx.Bool(LightNoSyncServeFlag.Name) } } @@ -1195,8 +1344,8 @@ func MakeAddress(ks *keystore.KeyStore, account string) (accounts.Account, error func setEtherbase(ctx *cli.Context, ks *keystore.KeyStore, cfg *ethconfig.Config) { // Extract the current etherbase var etherbase string - if ctx.GlobalIsSet(MinerEtherbaseFlag.Name) { - etherbase = ctx.GlobalString(MinerEtherbaseFlag.Name) + if ctx.IsSet(MinerEtherbaseFlag.Name) { + etherbase = ctx.String(MinerEtherbaseFlag.Name) } // Convert the etherbase into an address and configure it if etherbase != "" { @@ -1214,7 +1363,7 @@ func setEtherbase(ctx *cli.Context, ks *keystore.KeyStore, cfg *ethconfig.Config // MakePasswordList reads password lines from the file specified by the global --password flag. func MakePasswordList(ctx *cli.Context) []string { - path := ctx.GlobalString(PasswordFileFlag.Name) + path := ctx.Path(PasswordFileFlag.Name) if path == "" { return nil } @@ -1237,25 +1386,25 @@ func SetP2PConfig(ctx *cli.Context, cfg *p2p.Config) { setBootstrapNodes(ctx, cfg) setBootstrapNodesV5(ctx, cfg) - lightClient := ctx.GlobalString(SyncModeFlag.Name) == "light" - lightServer := (ctx.GlobalInt(LightServeFlag.Name) != 0) + lightClient := ctx.String(SyncModeFlag.Name) == "light" + lightServer := (ctx.Int(LightServeFlag.Name) != 0) - lightPeers := ctx.GlobalInt(LightMaxPeersFlag.Name) - if lightClient && !ctx.GlobalIsSet(LightMaxPeersFlag.Name) { + lightPeers := ctx.Int(LightMaxPeersFlag.Name) + if lightClient && !ctx.IsSet(LightMaxPeersFlag.Name) { // dynamic default - for clients we use 1/10th of the default for servers lightPeers /= 10 } - if ctx.GlobalIsSet(MaxPeersFlag.Name) { - cfg.MaxPeers = ctx.GlobalInt(MaxPeersFlag.Name) - if lightServer && !ctx.GlobalIsSet(LightMaxPeersFlag.Name) { + if ctx.IsSet(MaxPeersFlag.Name) { + cfg.MaxPeers = ctx.Int(MaxPeersFlag.Name) + if lightServer && !ctx.IsSet(LightMaxPeersFlag.Name) { cfg.MaxPeers += lightPeers } } else { if lightServer { cfg.MaxPeers += lightPeers } - if lightClient && ctx.GlobalIsSet(LightMaxPeersFlag.Name) && cfg.MaxPeers < lightPeers { + if lightClient && ctx.IsSet(LightMaxPeersFlag.Name) && cfg.MaxPeers < lightPeers { cfg.MaxPeers = lightPeers } } @@ -1268,24 +1417,24 @@ func SetP2PConfig(ctx *cli.Context, cfg *p2p.Config) { } log.Info("Maximum peer count", "ETH", ethPeers, "LES", lightPeers, "total", cfg.MaxPeers) - if ctx.GlobalIsSet(MaxPendingPeersFlag.Name) { - cfg.MaxPendingPeers = ctx.GlobalInt(MaxPendingPeersFlag.Name) + if ctx.IsSet(MaxPendingPeersFlag.Name) { + cfg.MaxPendingPeers = ctx.Int(MaxPendingPeersFlag.Name) } - if ctx.GlobalIsSet(NoDiscoverFlag.Name) || lightClient { + if ctx.IsSet(NoDiscoverFlag.Name) || lightClient { cfg.NoDiscovery = true } // if we're running a light client or server, force enable the v5 peer discovery // unless it is explicitly disabled with --nodiscover note that explicitly specifying // --v5disc overrides --nodiscover, in which case the later only disables v4 discovery - forceV5Discovery := (lightClient || lightServer) && !ctx.GlobalBool(NoDiscoverFlag.Name) - if ctx.GlobalIsSet(DiscoveryV5Flag.Name) { - cfg.DiscoveryV5 = ctx.GlobalBool(DiscoveryV5Flag.Name) + forceV5Discovery := (lightClient || lightServer) && !ctx.Bool(NoDiscoverFlag.Name) + if ctx.IsSet(DiscoveryV5Flag.Name) { + cfg.DiscoveryV5 = ctx.Bool(DiscoveryV5Flag.Name) } else if forceV5Discovery { cfg.DiscoveryV5 = true } - if netrestrict := ctx.GlobalString(NetrestrictFlag.Name); netrestrict != "" { + if netrestrict := ctx.String(NetrestrictFlag.Name); netrestrict != "" { list, err := netutil.ParseNetlist(netrestrict) if err != nil { Fatalf("Option %q: %v", NetrestrictFlag.Name, err) @@ -1293,7 +1442,7 @@ func SetP2PConfig(ctx *cli.Context, cfg *p2p.Config) { cfg.NetRestrict = list } - if ctx.GlobalBool(DeveloperFlag.Name) { + if ctx.Bool(DeveloperFlag.Name) { // --dev mode can't use p2p networking. cfg.MaxPeers = 0 cfg.ListenAddr = "" @@ -1311,40 +1460,40 @@ func SetNodeConfig(ctx *cli.Context, cfg *node.Config) { setGraphQL(ctx, cfg) setWS(ctx, cfg) setNodeUserIdent(ctx, cfg) - setDataDir(ctx, cfg) + SetDataDir(ctx, cfg) setSmartCard(ctx, cfg) - if ctx.GlobalIsSet(JWTSecretFlag.Name) { - cfg.JWTSecret = ctx.GlobalString(JWTSecretFlag.Name) + if ctx.IsSet(JWTSecretFlag.Name) { + cfg.JWTSecret = ctx.String(JWTSecretFlag.Name) } - if ctx.GlobalIsSet(ExternalSignerFlag.Name) { - cfg.ExternalSigner = ctx.GlobalString(ExternalSignerFlag.Name) + if ctx.IsSet(ExternalSignerFlag.Name) { + cfg.ExternalSigner = ctx.String(ExternalSignerFlag.Name) } - if ctx.GlobalIsSet(KeyStoreDirFlag.Name) { - cfg.KeyStoreDir = ctx.GlobalString(KeyStoreDirFlag.Name) + if ctx.IsSet(KeyStoreDirFlag.Name) { + cfg.KeyStoreDir = ctx.String(KeyStoreDirFlag.Name) } - if ctx.GlobalIsSet(DeveloperFlag.Name) { + if ctx.IsSet(DeveloperFlag.Name) { cfg.UseLightweightKDF = true } - if ctx.GlobalIsSet(LightKDFFlag.Name) { - cfg.UseLightweightKDF = ctx.GlobalBool(LightKDFFlag.Name) + if ctx.IsSet(LightKDFFlag.Name) { + cfg.UseLightweightKDF = ctx.Bool(LightKDFFlag.Name) } - if ctx.GlobalIsSet(NoUSBFlag.Name) || cfg.NoUSB { + if ctx.IsSet(NoUSBFlag.Name) || cfg.NoUSB { log.Warn("Option nousb is deprecated and USB is deactivated by default. Use --usb to enable") } - if ctx.GlobalIsSet(USBFlag.Name) { - cfg.USB = ctx.GlobalBool(USBFlag.Name) + if ctx.IsSet(USBFlag.Name) { + cfg.USB = ctx.Bool(USBFlag.Name) } - if ctx.GlobalIsSet(InsecureUnlockAllowedFlag.Name) { - cfg.InsecureUnlockAllowed = ctx.GlobalBool(InsecureUnlockAllowedFlag.Name) + if ctx.IsSet(InsecureUnlockAllowedFlag.Name) { + cfg.InsecureUnlockAllowed = ctx.Bool(InsecureUnlockAllowedFlag.Name) } } func setSmartCard(ctx *cli.Context, cfg *node.Config) { // Skip enabling smartcards if no path is set - path := ctx.GlobalString(SmartCardDaemonPathFlag.Name) + path := ctx.String(SmartCardDaemonPathFlag.Name) if path == "" { return } @@ -1362,13 +1511,13 @@ func setSmartCard(ctx *cli.Context, cfg *node.Config) { cfg.SmartCardDaemonPath = path } -func setDataDir(ctx *cli.Context, cfg *node.Config) { +func SetDataDir(ctx *cli.Context, cfg *node.Config) { switch { - case ctx.GlobalIsSet(DataDirFlag.Name): - cfg.DataDir = ctx.GlobalString(DataDirFlag.Name) - case ctx.GlobalBool(DeveloperFlag.Name): + case ctx.IsSet(DataDirFlag.Name): + cfg.DataDir = ctx.String(DataDirFlag.Name) + case ctx.Bool(DeveloperFlag.Name): cfg.DataDir = "" // unless explicitly requested, use memory databases - case ctx.GlobalBool(RopstenFlag.Name) && cfg.DataDir == node.DefaultDataDir(): + case ctx.Bool(RopstenFlag.Name) && cfg.DataDir == node.DefaultDataDir(): // Maintain compatibility with older Geth configurations storing the // Ropsten database in `testnet` instead of `ropsten`. legacyPath := filepath.Join(node.DefaultDataDir(), "testnet") @@ -1380,13 +1529,13 @@ func setDataDir(ctx *cli.Context, cfg *node.Config) { } cfg.DataDir = filepath.Join(node.DefaultDataDir(), "ropsten") - case ctx.GlobalBool(RinkebyFlag.Name) && cfg.DataDir == node.DefaultDataDir(): + case ctx.Bool(RinkebyFlag.Name) && cfg.DataDir == node.DefaultDataDir(): cfg.DataDir = filepath.Join(node.DefaultDataDir(), "rinkeby") - case ctx.GlobalBool(GoerliFlag.Name) && cfg.DataDir == node.DefaultDataDir(): + case ctx.Bool(GoerliFlag.Name) && cfg.DataDir == node.DefaultDataDir(): cfg.DataDir = filepath.Join(node.DefaultDataDir(), "goerli") - case ctx.GlobalBool(SepoliaFlag.Name) && cfg.DataDir == node.DefaultDataDir(): + case ctx.Bool(SepoliaFlag.Name) && cfg.DataDir == node.DefaultDataDir(): cfg.DataDir = filepath.Join(node.DefaultDataDir(), "sepolia") - case ctx.GlobalBool(KilnFlag.Name) && cfg.DataDir == node.DefaultDataDir(): + case ctx.Bool(KilnFlag.Name) && cfg.DataDir == node.DefaultDataDir(): cfg.DataDir = filepath.Join(node.DefaultDataDir(), "kiln") } } @@ -1397,23 +1546,23 @@ func setGPO(ctx *cli.Context, cfg *gasprice.Config, light bool) { if light { *cfg = ethconfig.LightClientGPO } - if ctx.GlobalIsSet(GpoBlocksFlag.Name) { - cfg.Blocks = ctx.GlobalInt(GpoBlocksFlag.Name) + if ctx.IsSet(GpoBlocksFlag.Name) { + cfg.Blocks = ctx.Int(GpoBlocksFlag.Name) } - if ctx.GlobalIsSet(GpoPercentileFlag.Name) { - cfg.Percentile = ctx.GlobalInt(GpoPercentileFlag.Name) + if ctx.IsSet(GpoPercentileFlag.Name) { + cfg.Percentile = ctx.Int(GpoPercentileFlag.Name) } - if ctx.GlobalIsSet(GpoMaxGasPriceFlag.Name) { - cfg.MaxPrice = big.NewInt(ctx.GlobalInt64(GpoMaxGasPriceFlag.Name)) + if ctx.IsSet(GpoMaxGasPriceFlag.Name) { + cfg.MaxPrice = big.NewInt(ctx.Int64(GpoMaxGasPriceFlag.Name)) } - if ctx.GlobalIsSet(GpoIgnoreGasPriceFlag.Name) { - cfg.IgnorePrice = big.NewInt(ctx.GlobalInt64(GpoIgnoreGasPriceFlag.Name)) + if ctx.IsSet(GpoIgnoreGasPriceFlag.Name) { + cfg.IgnorePrice = big.NewInt(ctx.Int64(GpoIgnoreGasPriceFlag.Name)) } } func setTxPool(ctx *cli.Context, cfg *core.TxPoolConfig) { - if ctx.GlobalIsSet(TxPoolLocalsFlag.Name) { - locals := strings.Split(ctx.GlobalString(TxPoolLocalsFlag.Name), ",") + if ctx.IsSet(TxPoolLocalsFlag.Name) { + locals := strings.Split(ctx.String(TxPoolLocalsFlag.Name), ",") for _, account := range locals { if trimmed := strings.TrimSpace(account); !common.IsHexAddress(trimmed) { Fatalf("Invalid account in --txpool.locals: %s", trimmed) @@ -1422,96 +1571,96 @@ func setTxPool(ctx *cli.Context, cfg *core.TxPoolConfig) { } } } - if ctx.GlobalIsSet(TxPoolNoLocalsFlag.Name) { - cfg.NoLocals = ctx.GlobalBool(TxPoolNoLocalsFlag.Name) + if ctx.IsSet(TxPoolNoLocalsFlag.Name) { + cfg.NoLocals = ctx.Bool(TxPoolNoLocalsFlag.Name) } - if ctx.GlobalIsSet(TxPoolJournalFlag.Name) { - cfg.Journal = ctx.GlobalString(TxPoolJournalFlag.Name) + if ctx.IsSet(TxPoolJournalFlag.Name) { + cfg.Journal = ctx.String(TxPoolJournalFlag.Name) } - if ctx.GlobalIsSet(TxPoolRejournalFlag.Name) { - cfg.Rejournal = ctx.GlobalDuration(TxPoolRejournalFlag.Name) + if ctx.IsSet(TxPoolRejournalFlag.Name) { + cfg.Rejournal = ctx.Duration(TxPoolRejournalFlag.Name) } - if ctx.GlobalIsSet(TxPoolPriceLimitFlag.Name) { - cfg.PriceLimit = ctx.GlobalUint64(TxPoolPriceLimitFlag.Name) + if ctx.IsSet(TxPoolPriceLimitFlag.Name) { + cfg.PriceLimit = ctx.Uint64(TxPoolPriceLimitFlag.Name) } - if ctx.GlobalIsSet(TxPoolPriceBumpFlag.Name) { - cfg.PriceBump = ctx.GlobalUint64(TxPoolPriceBumpFlag.Name) + if ctx.IsSet(TxPoolPriceBumpFlag.Name) { + cfg.PriceBump = ctx.Uint64(TxPoolPriceBumpFlag.Name) } - if ctx.GlobalIsSet(TxPoolAccountSlotsFlag.Name) { - cfg.AccountSlots = ctx.GlobalUint64(TxPoolAccountSlotsFlag.Name) + if ctx.IsSet(TxPoolAccountSlotsFlag.Name) { + cfg.AccountSlots = ctx.Uint64(TxPoolAccountSlotsFlag.Name) } - if ctx.GlobalIsSet(TxPoolGlobalSlotsFlag.Name) { - cfg.GlobalSlots = ctx.GlobalUint64(TxPoolGlobalSlotsFlag.Name) + if ctx.IsSet(TxPoolGlobalSlotsFlag.Name) { + cfg.GlobalSlots = ctx.Uint64(TxPoolGlobalSlotsFlag.Name) } - if ctx.GlobalIsSet(TxPoolAccountQueueFlag.Name) { - cfg.AccountQueue = ctx.GlobalUint64(TxPoolAccountQueueFlag.Name) + if ctx.IsSet(TxPoolAccountQueueFlag.Name) { + cfg.AccountQueue = ctx.Uint64(TxPoolAccountQueueFlag.Name) } - if ctx.GlobalIsSet(TxPoolGlobalQueueFlag.Name) { - cfg.GlobalQueue = ctx.GlobalUint64(TxPoolGlobalQueueFlag.Name) + if ctx.IsSet(TxPoolGlobalQueueFlag.Name) { + cfg.GlobalQueue = ctx.Uint64(TxPoolGlobalQueueFlag.Name) } - if ctx.GlobalIsSet(TxPoolLifetimeFlag.Name) { - cfg.Lifetime = ctx.GlobalDuration(TxPoolLifetimeFlag.Name) + if ctx.IsSet(TxPoolLifetimeFlag.Name) { + cfg.Lifetime = ctx.Duration(TxPoolLifetimeFlag.Name) } } func setEthash(ctx *cli.Context, cfg *ethconfig.Config) { - if ctx.GlobalIsSet(EthashCacheDirFlag.Name) { - cfg.Ethash.CacheDir = ctx.GlobalString(EthashCacheDirFlag.Name) + if ctx.IsSet(EthashCacheDirFlag.Name) { + cfg.Ethash.CacheDir = ctx.String(EthashCacheDirFlag.Name) } - if ctx.GlobalIsSet(EthashDatasetDirFlag.Name) { - cfg.Ethash.DatasetDir = ctx.GlobalString(EthashDatasetDirFlag.Name) + if ctx.IsSet(EthashDatasetDirFlag.Name) { + cfg.Ethash.DatasetDir = ctx.String(EthashDatasetDirFlag.Name) } - if ctx.GlobalIsSet(EthashCachesInMemoryFlag.Name) { - cfg.Ethash.CachesInMem = ctx.GlobalInt(EthashCachesInMemoryFlag.Name) + if ctx.IsSet(EthashCachesInMemoryFlag.Name) { + cfg.Ethash.CachesInMem = ctx.Int(EthashCachesInMemoryFlag.Name) } - if ctx.GlobalIsSet(EthashCachesOnDiskFlag.Name) { - cfg.Ethash.CachesOnDisk = ctx.GlobalInt(EthashCachesOnDiskFlag.Name) + if ctx.IsSet(EthashCachesOnDiskFlag.Name) { + cfg.Ethash.CachesOnDisk = ctx.Int(EthashCachesOnDiskFlag.Name) } - if ctx.GlobalIsSet(EthashCachesLockMmapFlag.Name) { - cfg.Ethash.CachesLockMmap = ctx.GlobalBool(EthashCachesLockMmapFlag.Name) + if ctx.IsSet(EthashCachesLockMmapFlag.Name) { + cfg.Ethash.CachesLockMmap = ctx.Bool(EthashCachesLockMmapFlag.Name) } - if ctx.GlobalIsSet(EthashDatasetsInMemoryFlag.Name) { - cfg.Ethash.DatasetsInMem = ctx.GlobalInt(EthashDatasetsInMemoryFlag.Name) + if ctx.IsSet(EthashDatasetsInMemoryFlag.Name) { + cfg.Ethash.DatasetsInMem = ctx.Int(EthashDatasetsInMemoryFlag.Name) } - if ctx.GlobalIsSet(EthashDatasetsOnDiskFlag.Name) { - cfg.Ethash.DatasetsOnDisk = ctx.GlobalInt(EthashDatasetsOnDiskFlag.Name) + if ctx.IsSet(EthashDatasetsOnDiskFlag.Name) { + cfg.Ethash.DatasetsOnDisk = ctx.Int(EthashDatasetsOnDiskFlag.Name) } - if ctx.GlobalIsSet(EthashDatasetsLockMmapFlag.Name) { - cfg.Ethash.DatasetsLockMmap = ctx.GlobalBool(EthashDatasetsLockMmapFlag.Name) + if ctx.IsSet(EthashDatasetsLockMmapFlag.Name) { + cfg.Ethash.DatasetsLockMmap = ctx.Bool(EthashDatasetsLockMmapFlag.Name) } } func setMiner(ctx *cli.Context, cfg *miner.Config) { - if ctx.GlobalIsSet(MinerNotifyFlag.Name) { - cfg.Notify = strings.Split(ctx.GlobalString(MinerNotifyFlag.Name), ",") + if ctx.IsSet(MinerNotifyFlag.Name) { + cfg.Notify = strings.Split(ctx.String(MinerNotifyFlag.Name), ",") } - cfg.NotifyFull = ctx.GlobalBool(MinerNotifyFullFlag.Name) - if ctx.GlobalIsSet(MinerExtraDataFlag.Name) { - cfg.ExtraData = []byte(ctx.GlobalString(MinerExtraDataFlag.Name)) + cfg.NotifyFull = ctx.Bool(MinerNotifyFullFlag.Name) + if ctx.IsSet(MinerExtraDataFlag.Name) { + cfg.ExtraData = []byte(ctx.String(MinerExtraDataFlag.Name)) } - if ctx.GlobalIsSet(MinerGasLimitFlag.Name) { - cfg.GasCeil = ctx.GlobalUint64(MinerGasLimitFlag.Name) + if ctx.IsSet(MinerGasLimitFlag.Name) { + cfg.GasCeil = ctx.Uint64(MinerGasLimitFlag.Name) } - if ctx.GlobalIsSet(MinerGasPriceFlag.Name) { - cfg.GasPrice = GlobalBig(ctx, MinerGasPriceFlag.Name) + if ctx.IsSet(MinerGasPriceFlag.Name) { + cfg.GasPrice = flags.GlobalBig(ctx, MinerGasPriceFlag.Name) } - if ctx.GlobalIsSet(MinerRecommitIntervalFlag.Name) { - cfg.Recommit = ctx.GlobalDuration(MinerRecommitIntervalFlag.Name) + if ctx.IsSet(MinerRecommitIntervalFlag.Name) { + cfg.Recommit = ctx.Duration(MinerRecommitIntervalFlag.Name) } - if ctx.GlobalIsSet(MinerNoVerifyFlag.Name) { - cfg.Noverify = ctx.GlobalBool(MinerNoVerifyFlag.Name) + if ctx.IsSet(MinerNoVerifyFlag.Name) { + cfg.Noverify = ctx.Bool(MinerNoVerifyFlag.Name) } - if ctx.GlobalIsSet(LegacyMinerGasTargetFlag.Name) { + if ctx.IsSet(LegacyMinerGasTargetFlag.Name) { log.Warn("The generic --miner.gastarget flag is deprecated and will be removed in the future!") } } func setRequiredBlocks(ctx *cli.Context, cfg *ethconfig.Config) { - requiredBlocks := ctx.GlobalString(EthRequiredBlocksFlag.Name) + requiredBlocks := ctx.String(EthRequiredBlocksFlag.Name) if requiredBlocks == "" { - if ctx.GlobalIsSet(LegacyWhitelistFlag.Name) { + if ctx.IsSet(LegacyWhitelistFlag.Name) { log.Warn("The flag --whitelist is deprecated and will be removed, please use --eth.requiredblocks") - requiredBlocks = ctx.GlobalString(LegacyWhitelistFlag.Name) + requiredBlocks = ctx.String(LegacyWhitelistFlag.Name) } else { return } @@ -1546,13 +1695,13 @@ func CheckExclusive(ctx *cli.Context, args ...interface{}) { panic(fmt.Sprintf("invalid argument, not cli.Flag type: %T", args[i])) } // Check if next arg extends current and expand its name if so - name := flag.GetName() + name := flag.Names()[0] if i+1 < len(args) { switch option := args[i+1].(type) { case string: // Extended flag check, make sure value set doesn't conflict with passed in option - if ctx.GlobalString(flag.GetName()) == option { + if ctx.String(flag.Names()[0]) == option { name += "=" + option set = append(set, "--"+name) } @@ -1566,7 +1715,7 @@ func CheckExclusive(ctx *cli.Context, args ...interface{}) { } } // Mark the flag if it's set - if ctx.GlobalIsSet(flag.GetName()) { + if ctx.IsSet(flag.Names()[0]) { set = append(set, "--"+name) } } @@ -1581,11 +1730,11 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { CheckExclusive(ctx, MainnetFlag, DeveloperFlag, RopstenFlag, RinkebyFlag, GoerliFlag, SepoliaFlag, KilnFlag) CheckExclusive(ctx, LightServeFlag, SyncModeFlag, "light") CheckExclusive(ctx, DeveloperFlag, ExternalSignerFlag) // Can't use both ephemeral unlocked and external signer - if ctx.GlobalString(GCModeFlag.Name) == "archive" && ctx.GlobalUint64(TxLookupLimitFlag.Name) != 0 { - ctx.GlobalSet(TxLookupLimitFlag.Name, "0") + if ctx.String(GCModeFlag.Name) == "archive" && ctx.Uint64(TxLookupLimitFlag.Name) != 0 { + ctx.Set(TxLookupLimitFlag.Name, "0") log.Warn("Disable transaction unindexing for archive node") } - if ctx.GlobalIsSet(LightServeFlag.Name) && ctx.GlobalUint64(TxLookupLimitFlag.Name) != 0 { + if ctx.IsSet(LightServeFlag.Name) && ctx.Uint64(TxLookupLimitFlag.Name) != 0 { log.Warn("LES server cannot serve old transaction status and cannot connect below les/4 protocol version if transaction lookup index is limited") } var ks *keystore.KeyStore @@ -1593,7 +1742,7 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { ks = keystores[0].(*keystore.KeyStore) } setEtherbase(ctx, ks, cfg) - setGPO(ctx, &cfg.GPO, ctx.GlobalString(SyncModeFlag.Name) == "light") + setGPO(ctx, &cfg.GPO, ctx.String(SyncModeFlag.Name) == "light") setTxPool(ctx, &cfg.TxPool) setEthash(ctx, cfg) setMiner(ctx, &cfg.Miner) @@ -1608,66 +1757,69 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { mem.Total = 2 * 1024 * 1024 * 1024 } allowance := int(mem.Total / 1024 / 1024 / 3) - if cache := ctx.GlobalInt(CacheFlag.Name); cache > allowance { + if cache := ctx.Int(CacheFlag.Name); cache > allowance { log.Warn("Sanitizing cache to Go's GC limits", "provided", cache, "updated", allowance) - ctx.GlobalSet(CacheFlag.Name, strconv.Itoa(allowance)) + ctx.Set(CacheFlag.Name, strconv.Itoa(allowance)) } } // Ensure Go's GC ignores the database cache for trigger percentage - cache := ctx.GlobalInt(CacheFlag.Name) + cache := ctx.Int(CacheFlag.Name) gogc := math.Max(20, math.Min(100, 100/(float64(cache)/1024))) log.Debug("Sanitizing Go's GC trigger", "percent", int(gogc)) godebug.SetGCPercent(int(gogc)) - if ctx.GlobalIsSet(SyncModeFlag.Name) { - cfg.SyncMode = *GlobalTextMarshaler(ctx, SyncModeFlag.Name).(*downloader.SyncMode) + if ctx.IsSet(SyncModeFlag.Name) { + cfg.SyncMode = *flags.GlobalTextMarshaler(ctx, SyncModeFlag.Name).(*downloader.SyncMode) } - if ctx.GlobalIsSet(NetworkIdFlag.Name) { - cfg.NetworkId = ctx.GlobalUint64(NetworkIdFlag.Name) + if ctx.IsSet(NetworkIdFlag.Name) { + cfg.NetworkId = ctx.Uint64(NetworkIdFlag.Name) } - if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheDatabaseFlag.Name) { - cfg.DatabaseCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheDatabaseFlag.Name) / 100 + if ctx.IsSet(CacheFlag.Name) || ctx.IsSet(CacheDatabaseFlag.Name) { + cfg.DatabaseCache = ctx.Int(CacheFlag.Name) * ctx.Int(CacheDatabaseFlag.Name) / 100 } - cfg.DatabaseHandles = MakeDatabaseHandles(ctx.GlobalInt(FDLimitFlag.Name)) - if ctx.GlobalIsSet(AncientFlag.Name) { - cfg.DatabaseFreezer = ctx.GlobalString(AncientFlag.Name) + cfg.DatabaseHandles = MakeDatabaseHandles(ctx.Int(FDLimitFlag.Name)) + if ctx.IsSet(AncientFlag.Name) { + cfg.DatabaseFreezer = ctx.String(AncientFlag.Name) } - if gcmode := ctx.GlobalString(GCModeFlag.Name); gcmode != "full" && gcmode != "archive" { + if gcmode := ctx.String(GCModeFlag.Name); gcmode != "full" && gcmode != "archive" { Fatalf("--%s must be either 'full' or 'archive'", GCModeFlag.Name) } - if ctx.GlobalIsSet(GCModeFlag.Name) { - cfg.NoPruning = ctx.GlobalString(GCModeFlag.Name) == "archive" + if ctx.IsSet(GCModeFlag.Name) { + cfg.NoPruning = ctx.String(GCModeFlag.Name) == "archive" } - if ctx.GlobalIsSet(CacheNoPrefetchFlag.Name) { - cfg.NoPrefetch = ctx.GlobalBool(CacheNoPrefetchFlag.Name) + if ctx.IsSet(CacheNoPrefetchFlag.Name) { + cfg.NoPrefetch = ctx.Bool(CacheNoPrefetchFlag.Name) } // Read the value from the flag no matter if it's set or not. - cfg.Preimages = ctx.GlobalBool(CachePreimagesFlag.Name) + cfg.Preimages = ctx.Bool(CachePreimagesFlag.Name) if cfg.NoPruning && !cfg.Preimages { cfg.Preimages = true log.Info("Enabling recording of key preimages since archive mode is used") } - if ctx.GlobalIsSet(TxLookupLimitFlag.Name) { - cfg.TxLookupLimit = ctx.GlobalUint64(TxLookupLimitFlag.Name) + if ctx.IsSet(TxLookupLimitFlag.Name) { + cfg.TxLookupLimit = ctx.Uint64(TxLookupLimitFlag.Name) + } + if ctx.IsSet(CacheFlag.Name) || ctx.IsSet(CacheTrieFlag.Name) { + cfg.TrieCleanCache = ctx.Int(CacheFlag.Name) * ctx.Int(CacheTrieFlag.Name) / 100 } - if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheTrieFlag.Name) { - cfg.TrieCleanCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheTrieFlag.Name) / 100 + if ctx.IsSet(CacheTrieJournalFlag.Name) { + cfg.TrieCleanCacheJournal = ctx.String(CacheTrieJournalFlag.Name) } - if ctx.GlobalIsSet(CacheTrieJournalFlag.Name) { - cfg.TrieCleanCacheJournal = ctx.GlobalString(CacheTrieJournalFlag.Name) + if ctx.IsSet(CacheTrieRejournalFlag.Name) { + cfg.TrieCleanCacheRejournal = ctx.Duration(CacheTrieRejournalFlag.Name) } - if ctx.GlobalIsSet(CacheTrieRejournalFlag.Name) { - cfg.TrieCleanCacheRejournal = ctx.GlobalDuration(CacheTrieRejournalFlag.Name) + if ctx.IsSet(CacheFlag.Name) || ctx.IsSet(CacheGCFlag.Name) { + cfg.TrieDirtyCache = ctx.Int(CacheFlag.Name) * ctx.Int(CacheGCFlag.Name) / 100 } - if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheGCFlag.Name) { - cfg.TrieDirtyCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100 + if ctx.IsSet(CacheFlag.Name) || ctx.IsSet(CacheSnapshotFlag.Name) { + cfg.SnapshotCache = ctx.Int(CacheFlag.Name) * ctx.Int(CacheSnapshotFlag.Name) / 100 } - if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheSnapshotFlag.Name) { - cfg.SnapshotCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheSnapshotFlag.Name) / 100 + if ctx.IsSet(CacheLogSizeFlag.Name) { + cfg.FilterLogCacheSize = ctx.Int(CacheLogSizeFlag.Name) } - if !ctx.GlobalBool(SnapshotFlag.Name) { + if !ctx.Bool(SnapshotFlag.Name) { // If snap-sync is requested, this flag is also required if cfg.SyncMode == downloader.SnapSync { log.Info("Snap sync requested, enabling --snapshot") @@ -1676,32 +1828,32 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { cfg.SnapshotCache = 0 // Disabled } } - if ctx.GlobalIsSet(DocRootFlag.Name) { - cfg.DocRoot = ctx.GlobalString(DocRootFlag.Name) + if ctx.IsSet(DocRootFlag.Name) { + cfg.DocRoot = ctx.String(DocRootFlag.Name) } - if ctx.GlobalIsSet(VMEnableDebugFlag.Name) { + if ctx.IsSet(VMEnableDebugFlag.Name) { // TODO(fjl): force-enable this in --dev mode - cfg.EnablePreimageRecording = ctx.GlobalBool(VMEnableDebugFlag.Name) + cfg.EnablePreimageRecording = ctx.Bool(VMEnableDebugFlag.Name) } - if ctx.GlobalIsSet(RPCGlobalGasCapFlag.Name) { - cfg.RPCGasCap = ctx.GlobalUint64(RPCGlobalGasCapFlag.Name) + if ctx.IsSet(RPCGlobalGasCapFlag.Name) { + cfg.RPCGasCap = ctx.Uint64(RPCGlobalGasCapFlag.Name) } if cfg.RPCGasCap != 0 { log.Info("Set global gas cap", "cap", cfg.RPCGasCap) } else { log.Info("Global gas cap disabled") } - if ctx.GlobalIsSet(RPCGlobalEVMTimeoutFlag.Name) { - cfg.RPCEVMTimeout = ctx.GlobalDuration(RPCGlobalEVMTimeoutFlag.Name) + if ctx.IsSet(RPCGlobalEVMTimeoutFlag.Name) { + cfg.RPCEVMTimeout = ctx.Duration(RPCGlobalEVMTimeoutFlag.Name) } - if ctx.GlobalIsSet(RPCGlobalTxFeeCapFlag.Name) { - cfg.RPCTxFeeCap = ctx.GlobalFloat64(RPCGlobalTxFeeCapFlag.Name) + if ctx.IsSet(RPCGlobalTxFeeCapFlag.Name) { + cfg.RPCTxFeeCap = ctx.Float64(RPCGlobalTxFeeCapFlag.Name) } - if ctx.GlobalIsSet(NoDiscoverFlag.Name) { + if ctx.IsSet(NoDiscoverFlag.Name) { cfg.EthDiscoveryURLs, cfg.SnapDiscoveryURLs = []string{}, []string{} - } else if ctx.GlobalIsSet(DNSDiscoveryFlag.Name) { - urls := ctx.GlobalString(DNSDiscoveryFlag.Name) + } else if ctx.IsSet(DNSDiscoveryFlag.Name) { + urls := ctx.String(DNSDiscoveryFlag.Name) if urls == "" { cfg.EthDiscoveryURLs = []string{} } else { @@ -1710,25 +1862,25 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { } // Override any default configs for hard coded networks. switch { - case ctx.GlobalBool(MainnetFlag.Name): - if !ctx.GlobalIsSet(NetworkIdFlag.Name) { + case ctx.Bool(MainnetFlag.Name): + if !ctx.IsSet(NetworkIdFlag.Name) { cfg.NetworkId = 1 } cfg.Genesis = core.DefaultGenesisBlock() SetDNSDiscoveryDefaults(cfg, params.MainnetGenesisHash) - case ctx.GlobalBool(RopstenFlag.Name): - if !ctx.GlobalIsSet(NetworkIdFlag.Name) { + case ctx.Bool(RopstenFlag.Name): + if !ctx.IsSet(NetworkIdFlag.Name) { cfg.NetworkId = 3 } cfg.Genesis = core.DefaultRopstenGenesisBlock() SetDNSDiscoveryDefaults(cfg, params.RopstenGenesisHash) - case ctx.GlobalBool(SepoliaFlag.Name): - if !ctx.GlobalIsSet(NetworkIdFlag.Name) { + case ctx.Bool(SepoliaFlag.Name): + if !ctx.IsSet(NetworkIdFlag.Name) { cfg.NetworkId = 11155111 } cfg.Genesis = core.DefaultSepoliaGenesisBlock() SetDNSDiscoveryDefaults(cfg, params.SepoliaGenesisHash) - case ctx.GlobalBool(RinkebyFlag.Name): + case ctx.Bool(RinkebyFlag.Name): log.Warn("") log.Warn("--------------------------------------------------------------------------------") log.Warn("Please note, Rinkeby has been deprecated. It will still work for the time being,") @@ -1739,25 +1891,25 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { log.Warn("--------------------------------------------------------------------------------") log.Warn("") - if !ctx.GlobalIsSet(NetworkIdFlag.Name) { + if !ctx.IsSet(NetworkIdFlag.Name) { cfg.NetworkId = 4 } cfg.Genesis = core.DefaultRinkebyGenesisBlock() SetDNSDiscoveryDefaults(cfg, params.RinkebyGenesisHash) - case ctx.GlobalBool(GoerliFlag.Name): - if !ctx.GlobalIsSet(NetworkIdFlag.Name) { + case ctx.Bool(GoerliFlag.Name): + if !ctx.IsSet(NetworkIdFlag.Name) { cfg.NetworkId = 5 } cfg.Genesis = core.DefaultGoerliGenesisBlock() SetDNSDiscoveryDefaults(cfg, params.GoerliGenesisHash) - case ctx.GlobalBool(KilnFlag.Name): - if !ctx.GlobalIsSet(NetworkIdFlag.Name) { + case ctx.Bool(KilnFlag.Name): + if !ctx.IsSet(NetworkIdFlag.Name) { cfg.NetworkId = 1337802 } cfg.Genesis = core.DefaultKilnGenesisBlock() SetDNSDiscoveryDefaults(cfg, params.KilnGenesisHash) - case ctx.GlobalBool(DeveloperFlag.Name): - if !ctx.GlobalIsSet(NetworkIdFlag.Name) { + case ctx.Bool(DeveloperFlag.Name): + if !ctx.IsSet(NetworkIdFlag.Name) { cfg.NetworkId = 1337 } cfg.SyncMode = downloader.FullSync @@ -1790,8 +1942,8 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { log.Info("Using developer account", "address", developer.Address) // Create a new developer genesis block or reuse existing one - cfg.Genesis = core.DeveloperGenesisBlock(uint64(ctx.GlobalInt(DeveloperPeriodFlag.Name)), ctx.GlobalUint64(DeveloperGasLimitFlag.Name), developer.Address) - if ctx.GlobalIsSet(DataDirFlag.Name) { + cfg.Genesis = core.DeveloperGenesisBlock(uint64(ctx.Int(DeveloperPeriodFlag.Name)), ctx.Uint64(DeveloperGasLimitFlag.Name), developer.Address) + if ctx.IsSet(DataDirFlag.Name) { // If datadir doesn't exist we need to open db in write-mode // so leveldb can create files. readonly := true @@ -1806,7 +1958,7 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { } chaindb.Close() } - if !ctx.GlobalIsSet(MinerGasPriceFlag.Name) { + if !ctx.IsSet(MinerGasPriceFlag.Name) { cfg.Miner.GasPrice = big.NewInt(1) } default: @@ -1842,10 +1994,8 @@ func RegisterEthService(stack *node.Node, cfg *ethconfig.Config) (ethapi.Backend Fatalf("Failed to register the Ethereum service: %v", err) } stack.RegisterAPIs(tracers.APIs(backend.ApiBackend)) - if backend.BlockChain().Config().TerminalTotalDifficulty != nil { - if err := lescatalyst.Register(stack, backend); err != nil { - Fatalf("Failed to register the catalyst service: %v", err) - } + if err := lescatalyst.Register(stack, backend); err != nil { + Fatalf("Failed to register the Engine API service: %v", err) } return backend.ApiBackend, nil } @@ -1859,48 +2009,59 @@ func RegisterEthService(stack *node.Node, cfg *ethconfig.Config) (ethapi.Backend Fatalf("Failed to create the LES server: %v", err) } } - if backend.BlockChain().Config().TerminalTotalDifficulty != nil { - if err := ethcatalyst.Register(stack, backend); err != nil { - Fatalf("Failed to register the catalyst service: %v", err) - } + if err := ethcatalyst.Register(stack, backend); err != nil { + Fatalf("Failed to register the Engine API service: %v", err) } stack.RegisterAPIs(tracers.APIs(backend.APIBackend)) return backend.APIBackend, backend } -// RegisterEthStatsService configures the Ethereum Stats daemon and adds it to -// the given node. +// RegisterEthStatsService configures the Ethereum Stats daemon and adds it to the node. func RegisterEthStatsService(stack *node.Node, backend ethapi.Backend, url string) { if err := ethstats.New(stack, backend, backend.Engine(), url); err != nil { Fatalf("Failed to register the Ethereum Stats service: %v", err) } } -// RegisterGraphQLService is a utility function to construct a new service and register it against a node. -func RegisterGraphQLService(stack *node.Node, backend ethapi.Backend, cfg node.Config) { - if err := graphql.New(stack, backend, cfg.GraphQLCors, cfg.GraphQLVirtualHosts); err != nil { +// RegisterGraphQLService adds the GraphQL API to the node. +func RegisterGraphQLService(stack *node.Node, backend ethapi.Backend, filterSystem *filters.FilterSystem, cfg *node.Config) { + err := graphql.New(stack, backend, filterSystem, cfg.GraphQLCors, cfg.GraphQLVirtualHosts) + if err != nil { Fatalf("Failed to register the GraphQL service: %v", err) } } +// RegisterFilterAPI adds the eth log filtering RPC API to the node. +func RegisterFilterAPI(stack *node.Node, backend ethapi.Backend, ethcfg *ethconfig.Config) *filters.FilterSystem { + isLightClient := ethcfg.SyncMode == downloader.LightSync + filterSystem := filters.NewFilterSystem(backend, filters.Config{ + LogCacheSize: ethcfg.FilterLogCacheSize, + }) + stack.RegisterAPIs([]rpc.API{{ + Namespace: "eth", + Service: filters.NewFilterAPI(filterSystem, isLightClient), + }}) + return filterSystem +} + func SetupMetrics(ctx *cli.Context) { if metrics.Enabled { log.Info("Enabling metrics collection") var ( - enableExport = ctx.GlobalBool(MetricsEnableInfluxDBFlag.Name) - enableExportV2 = ctx.GlobalBool(MetricsEnableInfluxDBV2Flag.Name) + enableExport = ctx.Bool(MetricsEnableInfluxDBFlag.Name) + enableExportV2 = ctx.Bool(MetricsEnableInfluxDBV2Flag.Name) ) if enableExport || enableExportV2 { CheckExclusive(ctx, MetricsEnableInfluxDBFlag, MetricsEnableInfluxDBV2Flag) - v1FlagIsSet := ctx.GlobalIsSet(MetricsInfluxDBUsernameFlag.Name) || - ctx.GlobalIsSet(MetricsInfluxDBPasswordFlag.Name) + v1FlagIsSet := ctx.IsSet(MetricsInfluxDBUsernameFlag.Name) || + ctx.IsSet(MetricsInfluxDBPasswordFlag.Name) - v2FlagIsSet := ctx.GlobalIsSet(MetricsInfluxDBTokenFlag.Name) || - ctx.GlobalIsSet(MetricsInfluxDBOrganizationFlag.Name) || - ctx.GlobalIsSet(MetricsInfluxDBBucketFlag.Name) + v2FlagIsSet := ctx.IsSet(MetricsInfluxDBTokenFlag.Name) || + ctx.IsSet(MetricsInfluxDBOrganizationFlag.Name) || + ctx.IsSet(MetricsInfluxDBBucketFlag.Name) if enableExport && v2FlagIsSet { Fatalf("Flags --influxdb.metrics.organization, --influxdb.metrics.token, --influxdb.metrics.bucket are only available for influxdb-v2") @@ -1910,32 +2071,32 @@ func SetupMetrics(ctx *cli.Context) { } var ( - endpoint = ctx.GlobalString(MetricsInfluxDBEndpointFlag.Name) - database = ctx.GlobalString(MetricsInfluxDBDatabaseFlag.Name) - username = ctx.GlobalString(MetricsInfluxDBUsernameFlag.Name) - password = ctx.GlobalString(MetricsInfluxDBPasswordFlag.Name) - - token = ctx.GlobalString(MetricsInfluxDBTokenFlag.Name) - bucket = ctx.GlobalString(MetricsInfluxDBBucketFlag.Name) - organization = ctx.GlobalString(MetricsInfluxDBOrganizationFlag.Name) + endpoint = ctx.String(MetricsInfluxDBEndpointFlag.Name) + database = ctx.String(MetricsInfluxDBDatabaseFlag.Name) + username = ctx.String(MetricsInfluxDBUsernameFlag.Name) + password = ctx.String(MetricsInfluxDBPasswordFlag.Name) + + token = ctx.String(MetricsInfluxDBTokenFlag.Name) + bucket = ctx.String(MetricsInfluxDBBucketFlag.Name) + organization = ctx.String(MetricsInfluxDBOrganizationFlag.Name) ) if enableExport { - tagsMap := SplitTagsFlag(ctx.GlobalString(MetricsInfluxDBTagsFlag.Name)) + tagsMap := SplitTagsFlag(ctx.String(MetricsInfluxDBTagsFlag.Name)) log.Info("Enabling metrics export to InfluxDB") go influxdb.InfluxDBWithTags(metrics.DefaultRegistry, 10*time.Second, endpoint, database, username, password, "geth.", tagsMap) } else if enableExportV2 { - tagsMap := SplitTagsFlag(ctx.GlobalString(MetricsInfluxDBTagsFlag.Name)) + tagsMap := SplitTagsFlag(ctx.String(MetricsInfluxDBTagsFlag.Name)) log.Info("Enabling metrics export to InfluxDB (v2)") go influxdb.InfluxDBV2WithTags(metrics.DefaultRegistry, 10*time.Second, endpoint, token, bucket, organization, "geth.", tagsMap) } - if ctx.GlobalIsSet(MetricsHTTPFlag.Name) { - address := fmt.Sprintf("%s:%d", ctx.GlobalString(MetricsHTTPFlag.Name), ctx.GlobalInt(MetricsPortFlag.Name)) + if ctx.IsSet(MetricsHTTPFlag.Name) { + address := fmt.Sprintf("%s:%d", ctx.String(MetricsHTTPFlag.Name), ctx.Int(MetricsPortFlag.Name)) log.Info("Enabling stand-alone metrics HTTP endpoint", "address", address) exp.Setup(address) } @@ -1962,20 +2123,20 @@ func SplitTagsFlag(tagsFlag string) map[string]string { // MakeChainDatabase open an LevelDB using the flags passed to the client and will hard crash if it fails. func MakeChainDatabase(ctx *cli.Context, stack *node.Node, readonly bool) ethdb.Database { var ( - cache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheDatabaseFlag.Name) / 100 - handles = MakeDatabaseHandles(ctx.GlobalInt(FDLimitFlag.Name)) + cache = ctx.Int(CacheFlag.Name) * ctx.Int(CacheDatabaseFlag.Name) / 100 + handles = MakeDatabaseHandles(ctx.Int(FDLimitFlag.Name)) err error chainDb ethdb.Database ) switch { - case ctx.GlobalIsSet(RemoteDBFlag.Name): - log.Info("Using remote db", "url", ctx.GlobalString(RemoteDBFlag.Name)) - chainDb, err = remotedb.New(ctx.GlobalString(RemoteDBFlag.Name)) - case ctx.GlobalString(SyncModeFlag.Name) == "light": + case ctx.IsSet(RemoteDBFlag.Name): + log.Info("Using remote db", "url", ctx.String(RemoteDBFlag.Name)) + chainDb, err = remotedb.New(ctx.String(RemoteDBFlag.Name)) + case ctx.String(SyncModeFlag.Name) == "light": chainDb, err = stack.OpenDatabase("lightchaindata", cache, handles, "", readonly) default: - chainDb, err = stack.OpenDatabaseWithFreezer("chaindata", cache, handles, ctx.GlobalString(AncientFlag.Name), "", readonly) + chainDb, err = stack.OpenDatabaseWithFreezer("chaindata", cache, handles, ctx.String(AncientFlag.Name), "", readonly) } if err != nil { Fatalf("Could not open database: %v", err) @@ -1986,19 +2147,19 @@ func MakeChainDatabase(ctx *cli.Context, stack *node.Node, readonly bool) ethdb. func MakeGenesis(ctx *cli.Context) *core.Genesis { var genesis *core.Genesis switch { - case ctx.GlobalBool(MainnetFlag.Name): + case ctx.Bool(MainnetFlag.Name): genesis = core.DefaultGenesisBlock() - case ctx.GlobalBool(RopstenFlag.Name): + case ctx.Bool(RopstenFlag.Name): genesis = core.DefaultRopstenGenesisBlock() - case ctx.GlobalBool(SepoliaFlag.Name): + case ctx.Bool(SepoliaFlag.Name): genesis = core.DefaultSepoliaGenesisBlock() - case ctx.GlobalBool(RinkebyFlag.Name): + case ctx.Bool(RinkebyFlag.Name): genesis = core.DefaultRinkebyGenesisBlock() - case ctx.GlobalBool(GoerliFlag.Name): + case ctx.Bool(GoerliFlag.Name): genesis = core.DefaultGoerliGenesisBlock() - case ctx.GlobalBool(KilnFlag.Name): + case ctx.Bool(KilnFlag.Name): genesis = core.DefaultKilnGenesisBlock() - case ctx.GlobalBool(DeveloperFlag.Name): + case ctx.Bool(DeveloperFlag.Name): Fatalf("Developer chains are ephemeral") } return genesis @@ -2015,36 +2176,36 @@ func MakeChain(ctx *cli.Context, stack *node.Node) (chain *core.BlockChain, chai var engine consensus.Engine ethashConf := ethconfig.Defaults.Ethash - if ctx.GlobalBool(FakePoWFlag.Name) { + if ctx.Bool(FakePoWFlag.Name) { ethashConf.PowMode = ethash.ModeFake } engine = ethconfig.CreateConsensusEngine(stack, config, ðashConf, nil, false, chainDb) - if gcmode := ctx.GlobalString(GCModeFlag.Name); gcmode != "full" && gcmode != "archive" { + if gcmode := ctx.String(GCModeFlag.Name); gcmode != "full" && gcmode != "archive" { Fatalf("--%s must be either 'full' or 'archive'", GCModeFlag.Name) } cache := &core.CacheConfig{ TrieCleanLimit: ethconfig.Defaults.TrieCleanCache, - TrieCleanNoPrefetch: ctx.GlobalBool(CacheNoPrefetchFlag.Name), + TrieCleanNoPrefetch: ctx.Bool(CacheNoPrefetchFlag.Name), TrieDirtyLimit: ethconfig.Defaults.TrieDirtyCache, - TrieDirtyDisabled: ctx.GlobalString(GCModeFlag.Name) == "archive", + TrieDirtyDisabled: ctx.String(GCModeFlag.Name) == "archive", TrieTimeLimit: ethconfig.Defaults.TrieTimeout, SnapshotLimit: ethconfig.Defaults.SnapshotCache, - Preimages: ctx.GlobalBool(CachePreimagesFlag.Name), + Preimages: ctx.Bool(CachePreimagesFlag.Name), } if cache.TrieDirtyDisabled && !cache.Preimages { cache.Preimages = true log.Info("Enabling recording of key preimages since archive mode is used") } - if !ctx.GlobalBool(SnapshotFlag.Name) { + if !ctx.Bool(SnapshotFlag.Name) { cache.SnapshotLimit = 0 // Disabled } - if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheTrieFlag.Name) { - cache.TrieCleanLimit = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheTrieFlag.Name) / 100 + if ctx.IsSet(CacheFlag.Name) || ctx.IsSet(CacheTrieFlag.Name) { + cache.TrieCleanLimit = ctx.Int(CacheFlag.Name) * ctx.Int(CacheTrieFlag.Name) / 100 } - if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheGCFlag.Name) { - cache.TrieDirtyLimit = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100 + if ctx.IsSet(CacheFlag.Name) || ctx.IsSet(CacheGCFlag.Name) { + cache.TrieDirtyLimit = ctx.Int(CacheFlag.Name) * ctx.Int(CacheGCFlag.Name) / 100 } - vmcfg := vm.Config{EnablePreimageRecording: ctx.GlobalBool(VMEnableDebugFlag.Name)} + vmcfg := vm.Config{EnablePreimageRecording: ctx.Bool(VMEnableDebugFlag.Name)} // TODO(rjl493456442) disable snapshot generation/wiping if the chain is read only. // Disable transaction indexing/unindexing by default. @@ -2059,38 +2220,14 @@ func MakeChain(ctx *cli.Context, stack *node.Node) (chain *core.BlockChain, chai // scripts to preload before starting. func MakeConsolePreloads(ctx *cli.Context) []string { // Skip preloading if there's nothing to preload - if ctx.GlobalString(PreloadJSFlag.Name) == "" { + if ctx.String(PreloadJSFlag.Name) == "" { return nil } // Otherwise resolve absolute paths and return them var preloads []string - for _, file := range strings.Split(ctx.GlobalString(PreloadJSFlag.Name), ",") { + for _, file := range strings.Split(ctx.String(PreloadJSFlag.Name), ",") { preloads = append(preloads, strings.TrimSpace(file)) } return preloads } - -// MigrateFlags sets the global flag from a local flag when it's set. -// This is a temporary function used for migrating old command/flags to the -// new format. -// -// e.g. geth account new --keystore /tmp/mykeystore --lightkdf -// -// is equivalent after calling this method with: -// -// geth --keystore /tmp/mykeystore --lightkdf account new -// -// This allows the use of the existing configuration functionality. -// When all flags are migrated this function can be removed and the existing -// configuration functionality must be changed that is uses local flags -func MigrateFlags(action func(ctx *cli.Context) error) func(*cli.Context) error { - return func(ctx *cli.Context) error { - for _, name := range ctx.FlagNames() { - if ctx.IsSet(name) { - ctx.GlobalSet(name, ctx.String(name)) - } - } - return action(ctx) - } -} diff --git a/cmd/utils/flags_legacy.go b/cmd/utils/flags_legacy.go index a0f64f609b77a..651c69bd04925 100644 --- a/cmd/utils/flags_legacy.go +++ b/cmd/utils/flags_legacy.go @@ -20,15 +20,15 @@ import ( "fmt" "github.com/ethereum/go-ethereum/eth/ethconfig" - "gopkg.in/urfave/cli.v1" + "github.com/ethereum/go-ethereum/internal/flags" + "github.com/urfave/cli/v2" ) -var ShowDeprecated = cli.Command{ +var ShowDeprecated = &cli.Command{ Action: showDeprecated, Name: "show-deprecated-flags", Usage: "Show flags that have been deprecated", ArgsUsage: " ", - Category: "MISCELLANEOUS COMMANDS", Description: "Show flags that have been deprecated and will soon be removed", } @@ -39,20 +39,22 @@ var DeprecatedFlags = []cli.Flag{ var ( // (Deprecated May 2020, shown in aliased flags section) - NoUSBFlag = cli.BoolFlag{ - Name: "nousb", - Usage: "Disables monitoring for and managing USB hardware wallets (deprecated)", + NoUSBFlag = &cli.BoolFlag{ + Name: "nousb", + Usage: "Disables monitoring for and managing USB hardware wallets (deprecated)", + Category: flags.DeprecatedCategory, } // (Deprecated July 2021, shown in aliased flags section) - LegacyMinerGasTargetFlag = cli.Uint64Flag{ - Name: "miner.gastarget", - Usage: "Target gas floor for mined blocks (deprecated)", - Value: ethconfig.Defaults.Miner.GasFloor, + LegacyMinerGasTargetFlag = &cli.Uint64Flag{ + Name: "miner.gastarget", + Usage: "Target gas floor for mined blocks (deprecated)", + Value: ethconfig.Defaults.Miner.GasFloor, + Category: flags.DeprecatedCategory, } ) // showDeprecated displays deprecated flags that will be soon removed from the codebase. -func showDeprecated(*cli.Context) { +func showDeprecated(*cli.Context) error { fmt.Println("--------------------------------------------------------------------") fmt.Println("The following flags are deprecated and will be removed in the future!") fmt.Println("--------------------------------------------------------------------") @@ -61,4 +63,5 @@ func showDeprecated(*cli.Context) { fmt.Println(flag.String()) } fmt.Println() + return nil } diff --git a/common/compiler/solidity.go b/common/compiler/solidity.go index ad8a44aa04ae6..9de94017c2edf 100644 --- a/common/compiler/solidity.go +++ b/common/compiler/solidity.go @@ -66,13 +66,16 @@ func ParseCombinedJSON(combinedJSON []byte, source string, languageVersion strin contracts := make(map[string]*Contract) for name, info := range output.Contracts { // Parse the individual compilation results. - var abi interface{} + var abi, userdoc, devdoc interface{} if err := json.Unmarshal([]byte(info.Abi), &abi); err != nil { return nil, fmt.Errorf("solc: error reading abi definition (%v)", err) } - var userdoc, devdoc interface{} - json.Unmarshal([]byte(info.Userdoc), &userdoc) - json.Unmarshal([]byte(info.Devdoc), &devdoc) + if err := json.Unmarshal([]byte(info.Userdoc), &userdoc); err != nil { + return nil, fmt.Errorf("solc: error reading userdoc definition (%v)", err) + } + if err := json.Unmarshal([]byte(info.Devdoc), &devdoc); err != nil { + return nil, fmt.Errorf("solc: error reading devdoc definition (%v)", err) + } contracts[name] = &Contract{ Code: "0x" + info.Bin, diff --git a/common/fdlimit/fdlimit_test.go b/common/fdlimit/fdlimit_test.go index 21362b8463a3f..9fd5e9fc3cbd6 100644 --- a/common/fdlimit/fdlimit_test.go +++ b/common/fdlimit/fdlimit_test.go @@ -17,7 +17,6 @@ package fdlimit import ( - "fmt" "testing" ) @@ -30,7 +29,7 @@ func TestFileDescriptorLimits(t *testing.T) { t.Fatal(err) } if hardlimit < target { - t.Skip(fmt.Sprintf("system limit is less than desired test target: %d < %d", hardlimit, target)) + t.Skipf("system limit is less than desired test target: %d < %d", hardlimit, target) } if limit, err := Current(); err != nil || limit <= 0 { diff --git a/common/format.go b/common/format.go index 6fc21af719236..7af41f52d5401 100644 --- a/common/format.go +++ b/common/format.go @@ -27,12 +27,12 @@ import ( // the unnecessary precision off from the formatted textual representation. type PrettyDuration time.Duration -var prettyDurationRe = regexp.MustCompile(`\.[0-9]+`) +var prettyDurationRe = regexp.MustCompile(`\.[0-9]{4,}`) // String implements the Stringer interface, allowing pretty printing of duration // values rounded to three decimals. func (d PrettyDuration) String() string { - label := fmt.Sprintf("%v", time.Duration(d)) + label := time.Duration(d).String() if match := prettyDurationRe.FindString(label); len(match) > 4 { label = strings.Replace(label, match, match[:4], 1) } diff --git a/common/math/big_test.go b/common/math/big_test.go index f896ec65becfd..803b5e1cc6174 100644 --- a/common/math/big_test.go +++ b/common/math/big_test.go @@ -171,7 +171,6 @@ func BenchmarkByteAt(b *testing.B) { } func BenchmarkByteAtOld(b *testing.B) { - bigint := MustParseBig256("0x18F8F8F1000111000110011100222004330052300000000000000000FEFCF3CC") for i := 0; i < b.N; i++ { PaddedBigBytes(bigint, 32) @@ -244,7 +243,6 @@ func TestBigEndianByteAt(t *testing.T) { if actual != test.exp { t.Fatalf("Expected [%v] %v:th byte to be %v, was %v.", test.x, test.y, test.exp, actual) } - } } func TestLittleEndianByteAt(t *testing.T) { @@ -277,7 +275,6 @@ func TestLittleEndianByteAt(t *testing.T) { if actual != test.exp { t.Fatalf("Expected [%v] %v:th byte to be %v, was %v.", test.x, test.y, test.exp, actual) } - } } diff --git a/common/mclock/simclock.go b/common/mclock/simclock.go index 766ca0f8736c0..f5ad3f8bc0aaa 100644 --- a/common/mclock/simclock.go +++ b/common/mclock/simclock.go @@ -58,7 +58,7 @@ func (s *Simulated) Run(d time.Duration) { s.mu.Lock() s.init() - end := s.now + AbsTime(d) + end := s.now.Add(d) var do []func() for len(s.scheduled) > 0 && s.scheduled[0].at <= end { ev := heap.Pop(&s.scheduled).(*simTimer) @@ -134,7 +134,7 @@ func (s *Simulated) AfterFunc(d time.Duration, fn func()) Timer { func (s *Simulated) schedule(d time.Duration, fn func()) *simTimer { s.init() - at := s.now + AbsTime(d) + at := s.now.Add(d) ev := &simTimer{do: fn, at: at, s: s} heap.Push(&s.scheduled, ev) s.cond.Broadcast() diff --git a/common/prque/lazyqueue.go b/common/prque/lazyqueue.go index 37c2f3bd42af5..6fdb6cc1b79f9 100644 --- a/common/prque/lazyqueue.go +++ b/common/prque/lazyqueue.go @@ -87,13 +87,13 @@ func (q *LazyQueue) Refresh() { // refresh re-evaluates items in the older queue and swaps the two queues func (q *LazyQueue) refresh(now mclock.AbsTime) { - q.maxUntil = now + mclock.AbsTime(q.period) + q.maxUntil = now.Add(q.period) for q.queue[0].Len() != 0 { q.Push(heap.Pop(q.queue[0]).(*item).value) } q.queue[0], q.queue[1] = q.queue[1], q.queue[0] q.indexOffset = 1 - q.indexOffset - q.maxUntil += mclock.AbsTime(q.period) + q.maxUntil = q.maxUntil.Add(q.period) } // Push adds an item to the queue @@ -163,7 +163,7 @@ func (q *LazyQueue) PopItem() interface{} { return i } -// Remove removes removes the item with the given index. +// Remove removes the item with the given index. func (q *LazyQueue) Remove(index int) interface{} { if index < 0 { return nil diff --git a/common/prque/prque.go b/common/prque/prque.go index 54c78b5fc2ba1..fb02e3418c28a 100755 --- a/common/prque/prque.go +++ b/common/prque/prque.go @@ -41,13 +41,13 @@ func (p *Prque) Push(data interface{}, priority int64) { heap.Push(p.cont, &item{data, priority}) } -// Peek returns the value with the greates priority but does not pop it off. +// Peek returns the value with the greatest priority but does not pop it off. func (p *Prque) Peek() (interface{}, int64) { item := p.cont.blocks[0][0] return item.value, item.priority } -// Pops the value with the greates priority off the stack and returns it. +// Pops the value with the greatest priority off the stack and returns it. // Currently no shrinking is done. func (p *Prque) Pop() (interface{}, int64) { item := heap.Pop(p.cont).(*item) diff --git a/common/size_test.go b/common/size_test.go index 0938d483c4bb0..28f053d39f5d5 100644 --- a/common/size_test.go +++ b/common/size_test.go @@ -25,6 +25,8 @@ func TestStorageSizeString(t *testing.T) { size StorageSize str string }{ + {2839274474874, "2.58 TiB"}, + {2458492810, "2.29 GiB"}, {2381273, "2.27 MiB"}, {2192, "2.14 KiB"}, {12, "12.00 B"}, @@ -36,3 +38,22 @@ func TestStorageSizeString(t *testing.T) { } } } + +func TestStorageSizeTerminalString(t *testing.T) { + tests := []struct { + size StorageSize + str string + }{ + {2839274474874, "2.58TiB"}, + {2458492810, "2.29GiB"}, + {2381273, "2.27MiB"}, + {2192, "2.14KiB"}, + {12, "12.00B"}, + } + + for _, test := range tests { + if test.size.TerminalString() != test.str { + t.Errorf("%f: got %q, want %q", float64(test.size), test.size.TerminalString(), test.str) + } + } +} diff --git a/common/types_test.go b/common/types_test.go index 318e985f870b4..94492278d84ac 100644 --- a/common/types_test.go +++ b/common/types_test.go @@ -155,7 +155,6 @@ func BenchmarkAddressHex(b *testing.B) { } func TestMixedcaseAccount_Address(t *testing.T) { - // https://github.com/ethereum/EIPs/blob/master/EIPS/eip-55.md // Note: 0X{checksum_addr} is not valid according to spec above @@ -192,9 +191,7 @@ func TestMixedcaseAccount_Address(t *testing.T) { if err := json.Unmarshal([]byte(r), &r2); err == nil { t.Errorf("Expected failure, input %v", r) } - } - } func TestHash_Scan(t *testing.T) { diff --git a/consensus/beacon/consensus.go b/consensus/beacon/consensus.go index 1fd7deb872fb4..7e4d657413eda 100644 --- a/consensus/beacon/consensus.go +++ b/consensus/beacon/consensus.go @@ -45,6 +45,7 @@ var ( errTooManyUncles = errors.New("too many uncles") errInvalidNonce = errors.New("invalid nonce") errInvalidUncleHash = errors.New("invalid uncle hash") + errInvalidTimestamp = errors.New("invalid timestamp") ) // Beacon is a consensus engine that combines the eth1 consensus and proof-of-stake @@ -78,7 +79,10 @@ func (beacon *Beacon) Author(header *types.Header) (common.Address, error) { // VerifyHeader checks whether a header conforms to the consensus rules of the // stock Ethereum consensus engine. func (beacon *Beacon) VerifyHeader(chain consensus.ChainHeaderReader, header *types.Header, seal bool) error { - reached, _ := IsTTDReached(chain, header.ParentHash, header.Number.Uint64()-1) + reached, err := IsTTDReached(chain, header.ParentHash, header.Number.Uint64()-1) + if err != nil { + return err + } if !reached { return beacon.ethone.VerifyHeader(chain, header, seal) } @@ -112,10 +116,23 @@ func (beacon *Beacon) VerifyHeaders(chain consensus.ChainHeaderReader, headers [ break } } - // All the headers have passed the transition point, use new rules. + if len(preHeaders) == 0 { + // All the headers are pos headers. Verify that the parent block reached total terminal difficulty. + if reached, err := IsTTDReached(chain, headers[0].ParentHash, headers[0].Number.Uint64()-1); !reached { + // TTD not reached for the first block, mark subsequent with invalid terminal block + if err == nil { + err = consensus.ErrInvalidTerminalBlock + } + results := make(chan error, len(headers)) + for i := 0; i < len(headers); i++ { + results <- err + } + return make(chan struct{}), results + } return beacon.verifyHeaders(chain, headers, nil) } + // The transition point exists in the middle, separate the headers // into two batches and apply different verification rules for them. var ( @@ -130,6 +147,14 @@ func (beacon *Beacon) VerifyHeaders(chain consensus.ChainHeaderReader, headers [ oldDone, oldResult = beacon.ethone.VerifyHeaders(chain, preHeaders, preSeals) newDone, newResult = beacon.verifyHeaders(chain, postHeaders, preHeaders[len(preHeaders)-1]) ) + // Verify that pre-merge headers don't overflow the TTD + if index, err := verifyTerminalPoWBlock(chain, preHeaders); err != nil { + // Mark all subsequent pow headers with the error. + for i := index; i < len(preHeaders); i++ { + errors[i], done[i] = err, true + } + } + // Collect the results for { for ; done[out]; out++ { results <- errors[out] @@ -139,7 +164,9 @@ func (beacon *Beacon) VerifyHeaders(chain consensus.ChainHeaderReader, headers [ } select { case err := <-oldResult: - errors[old], done[old] = err, true + if !done[old] { // skip TTD-verified failures + errors[old], done[old] = err, true + } old++ case err := <-newResult: errors[new], done[new] = err, true @@ -154,6 +181,33 @@ func (beacon *Beacon) VerifyHeaders(chain consensus.ChainHeaderReader, headers [ return abort, results } +// verifyTerminalPoWBlock verifies that the preHeaders conform to the specification +// wrt. their total difficulty. +// It expects: +// - preHeaders to be at least 1 element +// - the parent of the header element to be stored in the chain correctly +// - the preHeaders to have a set difficulty +// - the last element to be the terminal block +func verifyTerminalPoWBlock(chain consensus.ChainHeaderReader, preHeaders []*types.Header) (int, error) { + td := chain.GetTd(preHeaders[0].ParentHash, preHeaders[0].Number.Uint64()-1) + if td == nil { + return 0, consensus.ErrUnknownAncestor + } + td = new(big.Int).Set(td) + // Check that all blocks before the last one are below the TTD + for i, head := range preHeaders { + if td.Cmp(chain.Config().TerminalTotalDifficulty) >= 0 { + return i, consensus.ErrInvalidTerminalBlock + } + td.Add(td, head.Difficulty) + } + // Check that the last block is the terminal block + if td.Cmp(chain.Config().TerminalTotalDifficulty) < 0 { + return len(preHeaders) - 1, consensus.ErrInvalidTerminalBlock + } + return 0, nil +} + // VerifyUncles verifies that the given block's uncles conform to the consensus // rules of the Ethereum consensus engine. func (beacon *Beacon) VerifyUncles(chain consensus.ChainReader, block *types.Block) error { @@ -174,7 +228,7 @@ func (beacon *Beacon) VerifyUncles(chain consensus.ChainReader, block *types.Blo // - nonce is expected to be 0 // - unclehash is expected to be Hash(emptyHeader) // to be the desired constants -// (b) the timestamp is not verified anymore +// (b) we don't verify if a block is in the future anymore // (c) the extradata is limited to 32 bytes func (beacon *Beacon) verifyHeader(chain consensus.ChainHeaderReader, header, parent *types.Header) error { // Ensure that the header's extra-data section is of a reasonable size @@ -188,6 +242,10 @@ func (beacon *Beacon) verifyHeader(chain consensus.ChainHeaderReader, header, pa if header.UncleHash != types.EmptyUncleHash { return errInvalidUncleHash } + // Verify the timestamp + if header.Time <= parent.Time { + return errInvalidTimestamp + } // Verify the block's difficulty to ensure it's the default constant if beaconDifficulty.Cmp(header.Difficulty) != 0 { return fmt.Errorf("invalid difficulty: have %v, want %v", header.Difficulty, beaconDifficulty) diff --git a/consensus/beacon/consensus_test.go b/consensus/beacon/consensus_test.go new file mode 100644 index 0000000000000..09c0b27c42562 --- /dev/null +++ b/consensus/beacon/consensus_test.go @@ -0,0 +1,137 @@ +package beacon + +import ( + "fmt" + "math/big" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/params" +) + +type mockChain struct { + config *params.ChainConfig + tds map[uint64]*big.Int +} + +func newMockChain() *mockChain { + return &mockChain{ + config: new(params.ChainConfig), + tds: make(map[uint64]*big.Int), + } +} + +func (m *mockChain) Config() *params.ChainConfig { + return m.config +} + +func (m *mockChain) CurrentHeader() *types.Header { panic("not implemented") } + +func (m *mockChain) GetHeader(hash common.Hash, number uint64) *types.Header { + panic("not implemented") +} + +func (m *mockChain) GetHeaderByNumber(number uint64) *types.Header { panic("not implemented") } + +func (m *mockChain) GetHeaderByHash(hash common.Hash) *types.Header { panic("not implemented") } + +func (m *mockChain) GetTd(hash common.Hash, number uint64) *big.Int { + num, ok := m.tds[number] + if ok { + return new(big.Int).Set(num) + } + return nil +} + +func TestVerifyTerminalBlock(t *testing.T) { + chain := newMockChain() + chain.tds[0] = big.NewInt(10) + chain.config.TerminalTotalDifficulty = big.NewInt(50) + + tests := []struct { + preHeaders []*types.Header + ttd *big.Int + err error + index int + }{ + // valid ttd + { + preHeaders: []*types.Header{ + {Number: big.NewInt(1), Difficulty: big.NewInt(10)}, + {Number: big.NewInt(2), Difficulty: big.NewInt(10)}, + {Number: big.NewInt(3), Difficulty: big.NewInt(10)}, + {Number: big.NewInt(4), Difficulty: big.NewInt(10)}, + }, + ttd: big.NewInt(50), + }, + // last block doesn't reach ttd + { + preHeaders: []*types.Header{ + {Number: big.NewInt(1), Difficulty: big.NewInt(10)}, + {Number: big.NewInt(2), Difficulty: big.NewInt(10)}, + {Number: big.NewInt(3), Difficulty: big.NewInt(10)}, + {Number: big.NewInt(4), Difficulty: big.NewInt(9)}, + }, + ttd: big.NewInt(50), + err: consensus.ErrInvalidTerminalBlock, + index: 3, + }, + // two blocks reach ttd + { + preHeaders: []*types.Header{ + {Number: big.NewInt(1), Difficulty: big.NewInt(10)}, + {Number: big.NewInt(2), Difficulty: big.NewInt(10)}, + {Number: big.NewInt(3), Difficulty: big.NewInt(20)}, + {Number: big.NewInt(4), Difficulty: big.NewInt(10)}, + }, + ttd: big.NewInt(50), + err: consensus.ErrInvalidTerminalBlock, + index: 3, + }, + // three blocks reach ttd + { + preHeaders: []*types.Header{ + {Number: big.NewInt(1), Difficulty: big.NewInt(10)}, + {Number: big.NewInt(2), Difficulty: big.NewInt(10)}, + {Number: big.NewInt(3), Difficulty: big.NewInt(20)}, + {Number: big.NewInt(4), Difficulty: big.NewInt(10)}, + {Number: big.NewInt(4), Difficulty: big.NewInt(10)}, + }, + ttd: big.NewInt(50), + err: consensus.ErrInvalidTerminalBlock, + index: 3, + }, + // parent reached ttd + { + preHeaders: []*types.Header{ + {Number: big.NewInt(1), Difficulty: big.NewInt(10)}, + }, + ttd: big.NewInt(9), + err: consensus.ErrInvalidTerminalBlock, + index: 0, + }, + // unknown parent + { + preHeaders: []*types.Header{ + {Number: big.NewInt(4), Difficulty: big.NewInt(10)}, + }, + ttd: big.NewInt(9), + err: consensus.ErrUnknownAncestor, + index: 0, + }, + } + + for i, test := range tests { + fmt.Printf("Test: %v\n", i) + chain.config.TerminalTotalDifficulty = test.ttd + index, err := verifyTerminalPoWBlock(chain, test.preHeaders) + if err != test.err { + t.Fatalf("Invalid error encountered, expected %v got %v", test.err, err) + } + if index != test.index { + t.Fatalf("Invalid index, expected %v got %v", test.index, index) + } + } +} diff --git a/consensus/clique/clique.go b/consensus/clique/clique.go index 685186817d2d0..dcdfb20c6387a 100644 --- a/consensus/clique/clique.go +++ b/consensus/clique/clique.go @@ -180,7 +180,7 @@ type Clique struct { signer common.Address // Ethereum address of the signing key signFn SignerFn // Signer function to authorize hashes with - lock sync.RWMutex // Protects the signer fields + lock sync.RWMutex // Protects the signer and proposals fields // The fields below are for testing only fakeDiff bool // Skip difficulty verifications @@ -507,9 +507,8 @@ func (c *Clique) Prepare(chain consensus.ChainHeaderReader, header *types.Header if err != nil { return err } + c.lock.RLock() if number%c.config.Epoch != 0 { - c.lock.RLock() - // Gather all the proposals that make sense voting on addresses := make([]common.Address, 0, len(c.proposals)) for address, authorize := range c.proposals { @@ -526,10 +525,14 @@ func (c *Clique) Prepare(chain consensus.ChainHeaderReader, header *types.Header copy(header.Nonce[:], nonceDropVote) } } - c.lock.RUnlock() } + + // Copy signer protected by mutex to avoid race condition + signer := c.signer + c.lock.RUnlock() + // Set the correct difficulty - header.Difficulty = calcDifficulty(snap, c.signer) + header.Difficulty = calcDifficulty(snap, signer) // Ensure the extra data has all its components if len(header.Extra) < extraVanity { @@ -666,7 +669,10 @@ func (c *Clique) CalcDifficulty(chain consensus.ChainHeaderReader, time uint64, if err != nil { return nil } - return calcDifficulty(snap, c.signer) + c.lock.RLock() + signer := c.signer + c.lock.RUnlock() + return calcDifficulty(snap, signer) } func calcDifficulty(snap *Snapshot, signer common.Address) *big.Int { @@ -691,9 +697,7 @@ func (c *Clique) Close() error { func (c *Clique) APIs(chain consensus.ChainHeaderReader) []rpc.API { return []rpc.API{{ Namespace: "clique", - Version: "1.0", Service: &API{chain: chain, clique: c}, - Public: false, }} } diff --git a/consensus/clique/snapshot_test.go b/consensus/clique/snapshot_test.go index 094868ca744dd..4a067c62554a6 100644 --- a/consensus/clique/snapshot_test.go +++ b/consensus/clique/snapshot_test.go @@ -305,7 +305,7 @@ func TestClique(t *testing.T) { }, { // Ensure that pending votes don't survive authorization status changes. This // corner case can only appear if a signer is quickly added, removed and then - // readded (or the inverse), while one of the original voters dropped. If a + // re-added (or the inverse), while one of the original voters dropped. If a // past vote is left cached in the system somewhere, this will interfere with // the final signer outcome. signers: []string{"A", "B", "C", "D", "E"}, @@ -344,7 +344,7 @@ func TestClique(t *testing.T) { }, failure: errUnauthorizedSigner, }, { - // An authorized signer that signed recenty should not be able to sign again + // An authorized signer that signed recently should not be able to sign again signers: []string{"A", "B"}, votes: []testerVote{ {signer: "A"}, @@ -403,7 +403,7 @@ func TestClique(t *testing.T) { } // Create a pristine blockchain with the genesis injected db := rawdb.NewMemoryDatabase() - genesis.Commit(db) + genesisBlock := genesis.MustCommit(db) // Assemble a chain of headers from the cast votes config := *params.TestChainConfig @@ -414,7 +414,7 @@ func TestClique(t *testing.T) { engine := New(config.Clique, db) engine.fakeDiff = true - blocks, _ := core.GenerateChain(&config, genesis.ToBlock(db), engine, db, len(tt.votes), func(j int, gen *core.BlockGen) { + blocks, _ := core.GenerateChain(&config, genesisBlock, engine, db, len(tt.votes), func(j int, gen *core.BlockGen) { // Cast the vote contained in this block gen.SetCoinbase(accounts.address(tt.votes[j].voted)) if tt.votes[j].auth { diff --git a/consensus/errors.go b/consensus/errors.go index ac5242fb54c53..d508b6580f55b 100644 --- a/consensus/errors.go +++ b/consensus/errors.go @@ -34,4 +34,8 @@ var ( // ErrInvalidNumber is returned if a block's number doesn't equal its parent's // plus one. ErrInvalidNumber = errors.New("invalid block number") + + // ErrInvalidTerminalBlock is returned if a block is invalid wrt. the terminal + // total difficulty. + ErrInvalidTerminalBlock = errors.New("invalid terminal block") ) diff --git a/consensus/ethash/consensus.go b/consensus/ethash/consensus.go index 6a93fead29db6..1c38b80ea59b7 100644 --- a/consensus/ethash/consensus.go +++ b/consensus/ethash/consensus.go @@ -45,6 +45,11 @@ var ( maxUncles = 2 // Maximum number of uncles allowed in a single block allowedFutureBlockTimeSeconds = int64(15) // Max seconds from current time allowed for blocks, before they're considered future blocks + // calcDifficultyEip5133 is the difficulty adjustment algorithm as specified by EIP 5133. + // It offsets the bomb a total of 11.4M blocks. + // Specification EIP-5133: https://eips.ethereum.org/EIPS/eip-5133 + calcDifficultyEip5133 = makeDifficultyCalculator(big.NewInt(11_400_000)) + // calcDifficultyEip4345 is the difficulty adjustment algorithm as specified by EIP 4345. // It offsets the bomb a total of 10.7M blocks. // Specification EIP-4345: https://eips.ethereum.org/EIPS/eip-4345 @@ -334,6 +339,8 @@ func (ethash *Ethash) CalcDifficulty(chain consensus.ChainHeaderReader, time uin func CalcDifficulty(config *params.ChainConfig, time uint64, parent *types.Header) *big.Int { next := new(big.Int).Add(parent.Number, big1) switch { + case config.IsGrayGlacier(next): + return calcDifficultyEip5133(time, parent) case config.IsArrowGlacier(next): return calcDifficultyEip4345(time, parent) case config.IsLondon(next): diff --git a/consensus/ethash/consensus_test.go b/consensus/ethash/consensus_test.go index bca424af30e36..db997d737e625 100644 --- a/consensus/ethash/consensus_test.go +++ b/consensus/ethash/consensus_test.go @@ -103,7 +103,7 @@ func TestDifficultyCalculators(t *testing.T) { for i := 0; i < 5000; i++ { // 1 to 300 seconds diff var timeDelta = uint64(1 + rand.Uint32()%3000) - diffBig := big.NewInt(0).SetBytes(randSlice(2, 10)) + diffBig := new(big.Int).SetBytes(randSlice(2, 10)) if diffBig.Cmp(params.MinimumDifficulty) < 0 { diffBig.Set(params.MinimumDifficulty) } diff --git a/consensus/ethash/ethash.go b/consensus/ethash/ethash.go index c196ad0621704..dfe00d4b93c06 100644 --- a/consensus/ethash/ethash.go +++ b/consensus/ethash/ethash.go @@ -278,8 +278,11 @@ func (c *cache) generate(dir string, limit int, lock bool, test bool) { // Iterate over all previous instances and delete old ones for ep := int(c.epoch) - limit; ep >= 0; ep-- { seed := seedHash(uint64(ep)*epochLength + 1) - path := filepath.Join(dir, fmt.Sprintf("cache-R%d-%x%s", algorithmRevision, seed[:8], endian)) - os.Remove(path) + path := filepath.Join(dir, fmt.Sprintf("cache-R%d-%x%s*", algorithmRevision, seed[:8], endian)) + files, _ := filepath.Glob(path) // find also the temp files that are generated. + for _, file := range files { + os.Remove(file) + } } }) } @@ -678,15 +681,11 @@ func (ethash *Ethash) APIs(chain consensus.ChainHeaderReader) []rpc.API { return []rpc.API{ { Namespace: "eth", - Version: "1.0", Service: &API{ethash}, - Public: true, }, { Namespace: "ethash", - Version: "1.0", Service: &API{ethash}, - Public: true, }, } } diff --git a/consensus/misc/eip1559.go b/consensus/misc/eip1559.go index e18340b0f33ec..e0216243f0572 100644 --- a/consensus/misc/eip1559.go +++ b/consensus/misc/eip1559.go @@ -58,36 +58,36 @@ func CalcBaseFee(config *params.ChainConfig, parent *types.Header) *big.Int { return new(big.Int).SetUint64(params.InitialBaseFee) } - var ( - parentGasTarget = parent.GasLimit / params.ElasticityMultiplier - parentGasTargetBig = new(big.Int).SetUint64(parentGasTarget) - baseFeeChangeDenominator = new(big.Int).SetUint64(params.BaseFeeChangeDenominator) - ) + parentGasTarget := parent.GasLimit / params.ElasticityMultiplier // If the parent gasUsed is the same as the target, the baseFee remains unchanged. if parent.GasUsed == parentGasTarget { return new(big.Int).Set(parent.BaseFee) } + + var ( + num = new(big.Int) + denom = new(big.Int) + ) + if parent.GasUsed > parentGasTarget { // If the parent block used more gas than its target, the baseFee should increase. - gasUsedDelta := new(big.Int).SetUint64(parent.GasUsed - parentGasTarget) - x := new(big.Int).Mul(parent.BaseFee, gasUsedDelta) - y := x.Div(x, parentGasTargetBig) - baseFeeDelta := math.BigMax( - x.Div(y, baseFeeChangeDenominator), - common.Big1, - ) + // max(1, parentBaseFee * gasUsedDelta / parentGasTarget / baseFeeChangeDenominator) + num.SetUint64(parent.GasUsed - parentGasTarget) + num.Mul(num, parent.BaseFee) + num.Div(num, denom.SetUint64(parentGasTarget)) + num.Div(num, denom.SetUint64(params.BaseFeeChangeDenominator)) + baseFeeDelta := math.BigMax(num, common.Big1) - return x.Add(parent.BaseFee, baseFeeDelta) + return num.Add(parent.BaseFee, baseFeeDelta) } else { // Otherwise if the parent block used less gas than its target, the baseFee should decrease. - gasUsedDelta := new(big.Int).SetUint64(parentGasTarget - parent.GasUsed) - x := new(big.Int).Mul(parent.BaseFee, gasUsedDelta) - y := x.Div(x, parentGasTargetBig) - baseFeeDelta := x.Div(y, baseFeeChangeDenominator) + // max(0, parentBaseFee * gasUsedDelta / parentGasTarget / baseFeeChangeDenominator) + num.SetUint64(parentGasTarget - parent.GasUsed) + num.Mul(num, parent.BaseFee) + num.Div(num, denom.SetUint64(parentGasTarget)) + num.Div(num, denom.SetUint64(params.BaseFeeChangeDenominator)) + baseFee := num.Sub(parent.BaseFee, num) - return math.BigMax( - x.Sub(parent.BaseFee, baseFeeDelta), - common.Big0, - ) + return math.BigMax(baseFee, common.Big0) } } diff --git a/consensus/misc/forks.go b/consensus/misc/forks.go index 4a5e7c37e03c4..a6f3303ea6faa 100644 --- a/consensus/misc/forks.go +++ b/consensus/misc/forks.go @@ -35,7 +35,7 @@ func VerifyForkHashes(config *params.ChainConfig, header *types.Header, uncle bo // If the homestead reprice hash is set, validate it if config.EIP150Block != nil && config.EIP150Block.Cmp(header.Number) == 0 { if config.EIP150Hash != (common.Hash{}) && config.EIP150Hash != header.Hash() { - return fmt.Errorf("homestead gas reprice fork: have 0x%x, want 0x%x", header.Hash(), config.EIP150Hash) + return fmt.Errorf("homestead gas reprice fork: have %#x, want %#x", header.Hash(), config.EIP150Hash) } } // All ok, return diff --git a/console/console.go b/console/console.go index 2f61c1d7a4cf2..7b9ed27e15ecc 100644 --- a/console/console.go +++ b/console/console.go @@ -290,7 +290,7 @@ func (c *Console) AutoCompleteInput(line string, pos int) (string, []string, str if len(line) == 0 || pos == 0 { return "", nil, "" } - // Chunck data to relevant part for autocompletion + // Chunk data to relevant part for autocompletion // E.g. in case of nested lines eth.getBalance(eth.coinb start := pos - 1 for ; start > 0; start-- { @@ -407,7 +407,7 @@ func (c *Console) StopInteractive() { } } -// Interactive starts an interactive user session, where in.put is propted from +// Interactive starts an interactive user session, where input is prompted from // the configured user prompter. func (c *Console) Interactive() { var ( @@ -497,7 +497,7 @@ func (c *Console) readLines(input chan<- string, errc chan<- error, prompt <-cha } } -// countIndents returns the number of identations for the given input. +// countIndents returns the number of indentations for the given input. // In case of invalid input such as var a = } the result can be negative. func countIndents(input string) int { var ( @@ -540,11 +540,6 @@ func countIndents(input string) int { return indents } -// Execute runs the JavaScript file specified as the argument. -func (c *Console) Execute(path string) error { - return c.jsre.Exec(path) -} - // Stop cleans up the console and terminates the runtime environment. func (c *Console) Stop(graceful bool) error { c.stopOnce.Do(func() { diff --git a/console/console_test.go b/console/console_test.go index 04ba91d1576ad..35341fcba0b5d 100644 --- a/console/console_test.go +++ b/console/console_test.go @@ -234,19 +234,6 @@ func TestPreload(t *testing.T) { } } -// Tests that JavaScript scripts can be executes from the configured asset path. -func TestExecute(t *testing.T) { - tester := newTester(t, nil) - defer tester.Close(t) - - tester.console.Execute("exec.js") - - tester.console.Evaluate("execed") - if output := tester.output.String(); !strings.Contains(output, "some-executed-string") { - t.Fatalf("execed variable missing: have %s, want %s", output, "some-executed-string") - } -} - // Tests that the JavaScript objects returned by statement executions are properly // pretty printed instead of just displaying "[object]". func TestPrettyPrint(t *testing.T) { diff --git a/console/testdata/exec.js b/console/testdata/exec.js deleted file mode 100644 index 59e34d7c40334..0000000000000 --- a/console/testdata/exec.js +++ /dev/null @@ -1 +0,0 @@ -var execed = "some-executed-string"; diff --git a/contracts/checkpointoracle/oracle.go b/contracts/checkpointoracle/oracle.go index eff8ce2f2b803..dec01db24435c 100644 --- a/contracts/checkpointoracle/oracle.go +++ b/contracts/checkpointoracle/oracle.go @@ -17,7 +17,8 @@ // Package checkpointoracle is a an on-chain light client checkpoint oracle. package checkpointoracle -//go:generate go run ../../cmd/abigen --sol contract/oracle.sol --pkg contract --out contract/oracle.go +//go:generate solc contract/oracle.sol --combined-json bin,bin-runtime,srcmap,srcmap-runtime,abi,userdoc,devdoc,metadata,hashes --optimize -o ./ --overwrite +//go:generate go run ../../cmd/abigen --pkg contract --out contract/oracle.go --combined-json ./combined.json import ( "errors" diff --git a/core/asm/asm.go b/core/asm/asm.go index f3f129714d312..7c1e14ec01ead 100644 --- a/core/asm/asm.go +++ b/core/asm/asm.go @@ -109,7 +109,7 @@ func PrintDisassembled(code string) error { it := NewInstructionIterator(script) for it.Next() { if it.Arg() != nil && 0 < len(it.Arg()) { - fmt.Printf("%05x: %v 0x%x\n", it.PC(), it.Op(), it.Arg()) + fmt.Printf("%05x: %v %#x\n", it.PC(), it.Op(), it.Arg()) } else { fmt.Printf("%05x: %v\n", it.PC(), it.Op()) } @@ -124,7 +124,7 @@ func Disassemble(script []byte) ([]string, error) { it := NewInstructionIterator(script) for it.Next() { if it.Arg() != nil && 0 < len(it.Arg()) { - instrs = append(instrs, fmt.Sprintf("%05x: %v 0x%x\n", it.PC(), it.Op(), it.Arg())) + instrs = append(instrs, fmt.Sprintf("%05x: %v %#x\n", it.PC(), it.Op(), it.Arg())) } else { instrs = append(instrs, fmt.Sprintf("%05x: %v\n", it.PC(), it.Op())) } diff --git a/core/beacon/types.go b/core/beacon/types.go index 18d5d2ab78b46..e25d724c0d558 100644 --- a/core/beacon/types.go +++ b/core/beacon/types.go @@ -42,7 +42,7 @@ type payloadAttributesMarshaling struct { //go:generate go run github.com/fjl/gencodec -type ExecutableDataV1 -field-override executableDataMarshaling -out gen_ed.go -// ExecutableDataV1 structure described at https://github.com/ethereum/execution-apis/src/engine/specification.md +// ExecutableDataV1 structure described at https://github.com/ethereum/execution-apis/tree/main/src/engine/specification.md type ExecutableDataV1 struct { ParentHash common.Hash `json:"parentHash" gencodec:"required"` FeeRecipient common.Address `json:"feeRecipient" gencodec:"required"` @@ -148,6 +148,13 @@ func ExecutableDataToBlock(params ExecutableDataV1) (*types.Block, error) { if len(params.ExtraData) > 32 { return nil, fmt.Errorf("invalid extradata length: %v", len(params.ExtraData)) } + if len(params.LogsBloom) != 256 { + return nil, fmt.Errorf("invalid logsBloom length: %v", len(params.LogsBloom)) + } + // Check that baseFeePerGas is not negative or too big + if params.BaseFeePerGas != nil && (params.BaseFeePerGas.Sign() == -1 || params.BaseFeePerGas.BitLen() > 256) { + return nil, fmt.Errorf("invalid baseFeePerGas: %v", params.BaseFeePerGas) + } header := &types.Header{ ParentHash: params.ParentHash, UncleHash: types.EmptyUncleHash, diff --git a/core/block_validator_test.go b/core/block_validator_test.go index 0f183ba527784..8dee8d576070b 100644 --- a/core/block_validator_test.go +++ b/core/block_validator_test.go @@ -107,7 +107,8 @@ func testHeaderVerificationForMerging(t *testing.T, isClique bool) { Alloc: map[common.Address]GenesisAccount{ addr: {Balance: big.NewInt(1)}, }, - BaseFee: big.NewInt(params.InitialBaseFee), + BaseFee: big.NewInt(params.InitialBaseFee), + Difficulty: new(big.Int), } copy(genspec.ExtraData[32:], addr[:]) genesis := genspec.MustCommit(testdb) diff --git a/core/blockchain.go b/core/blockchain.go index 5ac12303cf732..a98c3b4dbeb34 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -51,6 +51,7 @@ var ( headHeaderGauge = metrics.NewRegisteredGauge("chain/head/header", nil) headFastBlockGauge = metrics.NewRegisteredGauge("chain/head/receipt", nil) headFinalizedBlockGauge = metrics.NewRegisteredGauge("chain/head/finalized", nil) + headSafeBlockGauge = metrics.NewRegisteredGauge("chain/head/safe", nil) accountReadTimer = metrics.NewRegisteredTimer("chain/account/reads", nil) accountHashTimer = metrics.NewRegisteredTimer("chain/account/hashes", nil) @@ -191,6 +192,7 @@ type BlockChain struct { currentBlock atomic.Value // Current head of the block chain currentFastBlock atomic.Value // Current head of the fast-sync chain (may be above the block chain!) currentFinalizedBlock atomic.Value // Current finalized head + currentSafeBlock atomic.Value // Current safe head stateCache state.Database // State database to reuse between imports (contains state cache) bodyCache *lru.Cache // Cache for the most recent block bodies @@ -267,6 +269,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par bc.currentBlock.Store(nilBlock) bc.currentFastBlock.Store(nilBlock) bc.currentFinalizedBlock.Store(nilBlock) + bc.currentSafeBlock.Store(nilBlock) // Initialize the chain with ancient data if it isn't empty. var txIndexBlock uint64 @@ -464,11 +467,15 @@ func (bc *BlockChain) loadLastState() error { } } - // Restore the last known finalized block + // Restore the last known finalized block and safe block + // Note: the safe block is not stored on disk and it is set to the last + // known finalized block on startup if head := rawdb.ReadFinalizedBlockHash(bc.db); head != (common.Hash{}) { if block := bc.GetBlockByHash(head); block != nil { bc.currentFinalizedBlock.Store(block) headFinalizedBlockGauge.Update(int64(block.NumberU64())) + bc.currentSafeBlock.Store(block) + headSafeBlockGauge.Update(int64(block.NumberU64())) } } // Issue a status log for the user @@ -504,8 +511,23 @@ func (bc *BlockChain) SetHead(head uint64) error { // SetFinalized sets the finalized block. func (bc *BlockChain) SetFinalized(block *types.Block) { bc.currentFinalizedBlock.Store(block) - rawdb.WriteFinalizedBlockHash(bc.db, block.Hash()) - headFinalizedBlockGauge.Update(int64(block.NumberU64())) + if block != nil { + rawdb.WriteFinalizedBlockHash(bc.db, block.Hash()) + headFinalizedBlockGauge.Update(int64(block.NumberU64())) + } else { + rawdb.WriteFinalizedBlockHash(bc.db, common.Hash{}) + headFinalizedBlockGauge.Update(0) + } +} + +// SetSafe sets the safe block. +func (bc *BlockChain) SetSafe(block *types.Block) { + bc.currentSafeBlock.Store(block) + if block != nil { + headSafeBlockGauge.Update(int64(block.NumberU64())) + } else { + headSafeBlockGauge.Update(0) + } } // setHeadBeyondRoot rewinds the local chain to a new head with the extra condition @@ -663,6 +685,16 @@ func (bc *BlockChain) setHeadBeyondRoot(head uint64, root common.Hash, repair bo bc.txLookupCache.Purge() bc.futureBlocks.Purge() + // Clear safe block, finalized block if needed + if safe := bc.CurrentSafeBlock(); safe != nil && head < safe.NumberU64() { + log.Warn("SetHead invalidated safe block") + bc.SetSafe(nil) + } + if finalized := bc.CurrentFinalizedBlock(); finalized != nil && head < finalized.NumberU64() { + log.Error("SetHead invalidated finalized block") + bc.SetFinalized(nil) + } + return rootNumber, bc.loadLastState() } @@ -674,7 +706,7 @@ func (bc *BlockChain) SnapSyncCommitHead(hash common.Hash) error { if block == nil { return fmt.Errorf("non existent block [%x..]", hash[:4]) } - if _, err := trie.NewSecure(block.Root(), bc.stateCache.TrieDB()); err != nil { + if _, err := trie.NewStateTrie(common.Hash{}, block.Root(), bc.stateCache.TrieDB()); err != nil { return err } @@ -739,22 +771,25 @@ func (bc *BlockChain) Export(w io.Writer) error { // ExportN writes a subset of the active chain to the given writer. func (bc *BlockChain) ExportN(w io.Writer, first uint64, last uint64) error { - if !bc.chainmu.TryLock() { - return errChainStopped - } - defer bc.chainmu.Unlock() - if first > last { return fmt.Errorf("export failed: first (%d) is greater than last (%d)", first, last) } log.Info("Exporting batch of blocks", "count", last-first+1) - start, reported := time.Now(), time.Now() + var ( + parentHash common.Hash + start = time.Now() + reported = time.Now() + ) for nr := first; nr <= last; nr++ { block := bc.GetBlockByNumber(nr) if block == nil { return fmt.Errorf("export failed on #%d: not found", nr) } + if nr > first && block.ParentHash() != parentHash { + return fmt.Errorf("export failed: chain reorg during export") + } + parentHash = block.Hash() if err := block.EncodeRLP(w); err != nil { return err } @@ -858,6 +893,10 @@ func (bc *BlockChain) Stop() { log.Error("Dangling trie nodes after full cleanup") } } + // Flush the collected preimages to disk + if err := bc.stateCache.TrieDB().CommitPreimages(); err != nil { + log.Error("Failed to commit trie preimages", "err", err) + } // Ensure all live cached entries be saved into disk, so that we can skip // cache warmup when node restarts. if bc.cacheConfig.TrieCleanJournal != "" { @@ -1209,7 +1248,7 @@ func (bc *BlockChain) writeKnownBlock(block *types.Block) error { // writeBlockWithState writes block, metadata and corresponding state data to the // database. -func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types.Receipt, logs []*types.Log, state *state.StateDB) error { +func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types.Receipt, state *state.StateDB) error { // Calculate the total difficulty of the block ptd := bc.GetTd(block.ParentHash(), block.NumberU64()-1) if ptd == nil { @@ -1304,7 +1343,7 @@ func (bc *BlockChain) WriteBlockAndSetHead(block *types.Block, receipts []*types // writeBlockAndSetHead is the internal implementation of WriteBlockAndSetHead. // This function expects the chain mutex to be held. func (bc *BlockChain) writeBlockAndSetHead(block *types.Block, receipts []*types.Receipt, logs []*types.Log, state *state.StateDB, emitHeadEvent bool) (status WriteStatus, err error) { - if err := bc.writeBlockWithState(block, receipts, logs, state); err != nil { + if err := bc.writeBlockWithState(block, receipts, state); err != nil { return NonStatTy, err } currentBlock := bc.CurrentBlock() @@ -1336,7 +1375,7 @@ func (bc *BlockChain) writeBlockAndSetHead(block *types.Block, receipts []*types } // In theory we should fire a ChainHeadEvent when we inject // a canonical block, but sometimes we can insert a batch of - // canonicial blocks. Avoid firing too many ChainHeadEvents, + // canonical blocks. Avoid firing too many ChainHeadEvents, // we will fire an accumulated ChainHeadEvent and disable fire // event here. if emitHeadEvent { @@ -1573,7 +1612,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals, setHead bool) // block in the middle. It can only happen in the clique chain. Whenever // we insert blocks via `insertSideChain`, we only commit `td`, `header` // and `body` if it's non-existent. Since we don't have receipts without - // reexecution, so nothing to commit. But if the sidechain will be adpoted + // reexecution, so nothing to commit. But if the sidechain will be adopted // as the canonical chain eventually, it needs to be reexecuted for missing // state, but if it's this special case here(skip reexecution) we will lose // the empty receipt entry. @@ -1668,7 +1707,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals, setHead bool) var status WriteStatus if !setHead { // Don't set the head, only insert the block - err = bc.writeBlockWithState(block, receipts, logs, statedb) + err = bc.writeBlockWithState(block, receipts, statedb) } else { status, err = bc.writeBlockAndSetHead(block, receipts, logs, statedb, false) } @@ -1965,8 +2004,8 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error { oldChain types.Blocks commonBlock *types.Block - deletedTxs types.Transactions - addedTxs types.Transactions + deletedTxs []common.Hash + addedTxs []common.Hash deletedLogs [][]*types.Log rebirthLogs [][]*types.Log @@ -1976,7 +2015,9 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error { // Old chain is longer, gather all transactions and logs as deleted ones for ; oldBlock != nil && oldBlock.NumberU64() != newBlock.NumberU64(); oldBlock = bc.GetBlock(oldBlock.ParentHash(), oldBlock.NumberU64()-1) { oldChain = append(oldChain, oldBlock) - deletedTxs = append(deletedTxs, oldBlock.Transactions()...) + for _, tx := range oldBlock.Transactions() { + deletedTxs = append(deletedTxs, tx.Hash()) + } // Collect deleted logs for notification logs := bc.collectLogs(oldBlock.Hash(), true) @@ -2006,7 +2047,9 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error { } // Remove an old block as well as stash away a new block oldChain = append(oldChain, oldBlock) - deletedTxs = append(deletedTxs, oldBlock.Transactions()...) + for _, tx := range oldBlock.Transactions() { + deletedTxs = append(deletedTxs, tx.Hash()) + } // Collect deleted logs for notification logs := bc.collectLogs(oldBlock.Hash(), true) @@ -2025,6 +2068,7 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error { return fmt.Errorf("invalid new chain") } } + // Ensure the user sees large reorgs if len(oldChain) > 0 && len(newChain) > 0 { logFn := log.Info @@ -2041,7 +2085,7 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error { } else if len(newChain) > 0 { // Special case happens in the post merge stage that current head is // the ancestor of new head while these two blocks are not consecutive - log.Info("Extend chain", "add", len(newChain), "number", newChain[0].NumberU64(), "hash", newChain[0].Hash()) + log.Info("Extend chain", "add", len(newChain), "number", newChain[0].Number(), "hash", newChain[0].Hash()) blockReorgAddMeter.Mark(int64(len(newChain))) } else { // len(newChain) == 0 && len(oldChain) > 0 @@ -2054,22 +2098,26 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error { // Insert the block in the canonical way, re-writing history bc.writeHeadBlock(newChain[i]) - // Collect reborn logs due to chain reorg - logs := bc.collectLogs(newChain[i].Hash(), false) - if len(logs) > 0 { - rebirthLogs = append(rebirthLogs, logs) - } // Collect the new added transactions. - addedTxs = append(addedTxs, newChain[i].Transactions()...) + for _, tx := range newChain[i].Transactions() { + addedTxs = append(addedTxs, tx.Hash()) + } } + // Delete useless indexes right now which includes the non-canonical // transaction indexes, canonical chain indexes which above the head. indexesBatch := bc.db.NewBatch() - for _, tx := range types.TxDifference(deletedTxs, addedTxs) { - rawdb.DeleteTxLookupEntry(indexesBatch, tx.Hash()) + for _, tx := range types.HashDifference(deletedTxs, addedTxs) { + rawdb.DeleteTxLookupEntry(indexesBatch, tx) + } + + // Delete all hash markers that are not part of the new canonical chain. + // Because the reorg function does not handle new chain head, all hash + // markers greater than or equal to new chain head should be deleted. + number := commonBlock.NumberU64() + if len(newChain) > 1 { + number = newChain[1].NumberU64() } - // Delete any canonical number assignments above the new head - number := bc.CurrentBlock().NumberU64() for i := number + 1; ; i++ { hash := rawdb.ReadCanonicalHash(bc.db, i) if hash == (common.Hash{}) { @@ -2080,6 +2128,15 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error { if err := indexesBatch.Write(); err != nil { log.Crit("Failed to delete useless indexes", "err", err) } + + // Collect the logs + for i := len(newChain) - 1; i >= 1; i-- { + // Collect reborn logs due to chain reorg + logs := bc.collectLogs(newChain[i].Hash(), false) + if len(logs) > 0 { + rebirthLogs = append(rebirthLogs, logs) + } + } // If any logs need to be fired, do it now. In theory we could avoid creating // this goroutine if there are no events to fire, but realistcally that only // ever happens if we're reorging empty blocks, which will only happen on idle @@ -2324,7 +2381,7 @@ func (bc *BlockChain) reportBlock(block *types.Block, receipts types.Receipts, e Chain config: %v Number: %v -Hash: 0x%x +Hash: %#x %v Error: %v @@ -2356,3 +2413,11 @@ func (bc *BlockChain) InsertHeaderChain(chain []*types.Header, checkFreq int) (i _, err := bc.hc.InsertHeaderChain(chain, start, bc.forker) return 0, err } + +// SetBlockValidatorAndProcessorForTesting sets the current validator and processor. +// This method can be used to force an invalid blockchain to be verified for tests. +// This method is unsafe and should only be used before block import starts. +func (bc *BlockChain) SetBlockValidatorAndProcessorForTesting(v Validator, p Processor) { + bc.validator = v + bc.processor = p +} diff --git a/core/blockchain_reader.go b/core/blockchain_reader.go index b8d4233c6ecd8..5814c8a0daee3 100644 --- a/core/blockchain_reader.go +++ b/core/blockchain_reader.go @@ -55,6 +55,12 @@ func (bc *BlockChain) CurrentFinalizedBlock() *types.Block { return bc.currentFinalizedBlock.Load().(*types.Block) } +// CurrentSafeBlock retrieves the current safe block of the canonical +// chain. The block is retrieved from the blockchain's internal cache. +func (bc *BlockChain) CurrentSafeBlock() *types.Block { + return bc.currentSafeBlock.Load().(*types.Block) +} + // HasHeader checks if a block header is present in the database or not, caching // it if present. func (bc *BlockChain) HasHeader(hash common.Hash, number uint64) bool { @@ -131,6 +137,9 @@ func (bc *BlockChain) HasBlock(hash common.Hash, number uint64) bool { if bc.blockCache.Contains(hash) { return true } + if !bc.HasHeader(hash, number) { + return false + } return rawdb.HasBody(bc.db, hash, number) } diff --git a/core/blockchain_repair_test.go b/core/blockchain_repair_test.go index 24309405d2b3a..feed8a1777890 100644 --- a/core/blockchain_repair_test.go +++ b/core/blockchain_repair_test.go @@ -564,7 +564,7 @@ func testShortReorgedSnapSyncingRepair(t *testing.T, snapshots bool) { // Tests a recovery for a long canonical chain with frozen blocks where a recent // block - newer than the ancient limit - was already committed to disk and then // the process crashed. In this case we expect the chain to be rolled back to the -// committed block, with everything afterwads kept as fast sync data. +// committed block, with everything afterwards kept as fast sync data. func TestLongShallowRepair(t *testing.T) { testLongShallowRepair(t, false) } func TestLongShallowRepairWithSnapshots(t *testing.T) { testLongShallowRepair(t, true) } @@ -609,7 +609,7 @@ func testLongShallowRepair(t *testing.T, snapshots bool) { // Tests a recovery for a long canonical chain with frozen blocks where a recent // block - older than the ancient limit - was already committed to disk and then // the process crashed. In this case we expect the chain to be rolled back to the -// committed block, with everything afterwads deleted. +// committed block, with everything afterwards deleted. func TestLongDeepRepair(t *testing.T) { testLongDeepRepair(t, false) } func TestLongDeepRepairWithSnapshots(t *testing.T) { testLongDeepRepair(t, true) } @@ -653,7 +653,7 @@ func testLongDeepRepair(t *testing.T, snapshots bool) { // Tests a recovery for a long canonical chain with frozen blocks where the fast // sync pivot point - newer than the ancient limit - was already committed, after // which the process crashed. In this case we expect the chain to be rolled back -// to the committed block, with everything afterwads kept as fast sync data. +// to the committed block, with everything afterwards kept as fast sync data. func TestLongSnapSyncedShallowRepair(t *testing.T) { testLongSnapSyncedShallowRepair(t, false) } @@ -702,7 +702,7 @@ func testLongSnapSyncedShallowRepair(t *testing.T, snapshots bool) { // Tests a recovery for a long canonical chain with frozen blocks where the fast // sync pivot point - older than the ancient limit - was already committed, after // which the process crashed. In this case we expect the chain to be rolled back -// to the committed block, with everything afterwads deleted. +// to the committed block, with everything afterwards deleted. func TestLongSnapSyncedDeepRepair(t *testing.T) { testLongSnapSyncedDeepRepair(t, false) } func TestLongSnapSyncedDeepRepairWithSnapshots(t *testing.T) { testLongSnapSyncedDeepRepair(t, true) } @@ -843,7 +843,7 @@ func testLongSnapSyncingDeepRepair(t *testing.T, snapshots bool) { // side chain, where a recent block - newer than the ancient limit - was already // committed to disk and then the process crashed. In this test scenario the side // chain is below the committed block. In this case we expect the chain to be -// rolled back to the committed block, with everything afterwads kept as fast +// rolled back to the committed block, with everything afterwards kept as fast // sync data; the side chain completely nuked by the freezer. func TestLongOldForkedShallowRepair(t *testing.T) { testLongOldForkedShallowRepair(t, false) @@ -895,7 +895,7 @@ func testLongOldForkedShallowRepair(t *testing.T, snapshots bool) { // side chain, where a recent block - older than the ancient limit - was already // committed to disk and then the process crashed. In this test scenario the side // chain is below the committed block. In this case we expect the canonical chain -// to be rolled back to the committed block, with everything afterwads deleted; +// to be rolled back to the committed block, with everything afterwards deleted; // the side chain completely nuked by the freezer. func TestLongOldForkedDeepRepair(t *testing.T) { testLongOldForkedDeepRepair(t, false) } func TestLongOldForkedDeepRepairWithSnapshots(t *testing.T) { testLongOldForkedDeepRepair(t, true) } @@ -942,7 +942,7 @@ func testLongOldForkedDeepRepair(t *testing.T, snapshots bool) { // side chain, where the fast sync pivot point - newer than the ancient limit - // was already committed to disk and then the process crashed. In this test scenario // the side chain is below the committed block. In this case we expect the chain -// to be rolled back to the committed block, with everything afterwads kept as +// to be rolled back to the committed block, with everything afterwards kept as // fast sync data; the side chain completely nuked by the freezer. func TestLongOldForkedSnapSyncedShallowRepair(t *testing.T) { testLongOldForkedSnapSyncedShallowRepair(t, false) @@ -994,7 +994,7 @@ func testLongOldForkedSnapSyncedShallowRepair(t *testing.T, snapshots bool) { // side chain, where the fast sync pivot point - older than the ancient limit - // was already committed to disk and then the process crashed. In this test scenario // the side chain is below the committed block. In this case we expect the canonical -// chain to be rolled back to the committed block, with everything afterwads deleted; +// chain to be rolled back to the committed block, with everything afterwards deleted; // the side chain completely nuked by the freezer. func TestLongOldForkedSnapSyncedDeepRepair(t *testing.T) { testLongOldForkedSnapSyncedDeepRepair(t, false) @@ -1149,7 +1149,7 @@ func testLongOldForkedSnapSyncingDeepRepair(t *testing.T, snapshots bool) { // side chain, where a recent block - newer than the ancient limit - was already // committed to disk and then the process crashed. In this test scenario the side // chain is above the committed block. In this case we expect the chain to be -// rolled back to the committed block, with everything afterwads kept as fast +// rolled back to the committed block, with everything afterwards kept as fast // sync data; the side chain completely nuked by the freezer. func TestLongNewerForkedShallowRepair(t *testing.T) { testLongNewerForkedShallowRepair(t, false) @@ -1201,7 +1201,7 @@ func testLongNewerForkedShallowRepair(t *testing.T, snapshots bool) { // side chain, where a recent block - older than the ancient limit - was already // committed to disk and then the process crashed. In this test scenario the side // chain is above the committed block. In this case we expect the canonical chain -// to be rolled back to the committed block, with everything afterwads deleted; +// to be rolled back to the committed block, with everything afterwards deleted; // the side chain completely nuked by the freezer. func TestLongNewerForkedDeepRepair(t *testing.T) { testLongNewerForkedDeepRepair(t, false) } func TestLongNewerForkedDeepRepairWithSnapshots(t *testing.T) { testLongNewerForkedDeepRepair(t, true) } @@ -1248,7 +1248,7 @@ func testLongNewerForkedDeepRepair(t *testing.T, snapshots bool) { // side chain, where the fast sync pivot point - newer than the ancient limit - // was already committed to disk and then the process crashed. In this test scenario // the side chain is above the committed block. In this case we expect the chain -// to be rolled back to the committed block, with everything afterwads kept as fast +// to be rolled back to the committed block, with everything afterwards kept as fast // sync data; the side chain completely nuked by the freezer. func TestLongNewerForkedSnapSyncedShallowRepair(t *testing.T) { testLongNewerForkedSnapSyncedShallowRepair(t, false) @@ -1300,7 +1300,7 @@ func testLongNewerForkedSnapSyncedShallowRepair(t *testing.T, snapshots bool) { // side chain, where the fast sync pivot point - older than the ancient limit - // was already committed to disk and then the process crashed. In this test scenario // the side chain is above the committed block. In this case we expect the canonical -// chain to be rolled back to the committed block, with everything afterwads deleted; +// chain to be rolled back to the committed block, with everything afterwards deleted; // the side chain completely nuked by the freezer. func TestLongNewerForkedSnapSyncedDeepRepair(t *testing.T) { testLongNewerForkedSnapSyncedDeepRepair(t, false) @@ -1454,7 +1454,7 @@ func testLongNewerForkedSnapSyncingDeepRepair(t *testing.T, snapshots bool) { // Tests a recovery for a long canonical chain with frozen blocks and a longer side // chain, where a recent block - newer than the ancient limit - was already committed // to disk and then the process crashed. In this case we expect the chain to be -// rolled back to the committed block, with everything afterwads kept as fast sync +// rolled back to the committed block, with everything afterwards kept as fast sync // data. The side chain completely nuked by the freezer. func TestLongReorgedShallowRepair(t *testing.T) { testLongReorgedShallowRepair(t, false) } func TestLongReorgedShallowRepairWithSnapshots(t *testing.T) { testLongReorgedShallowRepair(t, true) } @@ -1501,7 +1501,7 @@ func testLongReorgedShallowRepair(t *testing.T, snapshots bool) { // Tests a recovery for a long canonical chain with frozen blocks and a longer side // chain, where a recent block - older than the ancient limit - was already committed // to disk and then the process crashed. In this case we expect the canonical chains -// to be rolled back to the committed block, with everything afterwads deleted. The +// to be rolled back to the committed block, with everything afterwards deleted. The // side chain completely nuked by the freezer. func TestLongReorgedDeepRepair(t *testing.T) { testLongReorgedDeepRepair(t, false) } func TestLongReorgedDeepRepairWithSnapshots(t *testing.T) { testLongReorgedDeepRepair(t, true) } @@ -1548,7 +1548,7 @@ func testLongReorgedDeepRepair(t *testing.T, snapshots bool) { // side chain, where the fast sync pivot point - newer than the ancient limit - // was already committed to disk and then the process crashed. In this case we // expect the chain to be rolled back to the committed block, with everything -// afterwads kept as fast sync data. The side chain completely nuked by the +// afterwards kept as fast sync data. The side chain completely nuked by the // freezer. func TestLongReorgedSnapSyncedShallowRepair(t *testing.T) { testLongReorgedSnapSyncedShallowRepair(t, false) @@ -1600,7 +1600,7 @@ func testLongReorgedSnapSyncedShallowRepair(t *testing.T, snapshots bool) { // side chain, where the fast sync pivot point - older than the ancient limit - // was already committed to disk and then the process crashed. In this case we // expect the canonical chains to be rolled back to the committed block, with -// everything afterwads deleted. The side chain completely nuked by the freezer. +// everything afterwards deleted. The side chain completely nuked by the freezer. func TestLongReorgedSnapSyncedDeepRepair(t *testing.T) { testLongReorgedSnapSyncedDeepRepair(t, false) } diff --git a/core/blockchain_sethead_test.go b/core/blockchain_sethead_test.go index 970e0306308dd..d8103b5b5c50e 100644 --- a/core/blockchain_sethead_test.go +++ b/core/blockchain_sethead_test.go @@ -51,6 +51,7 @@ type rewindTest struct { expHeadBlock uint64 // Block number of the expected head full block } +//nolint:unused func (tt *rewindTest) dump(crash bool) string { buffer := new(strings.Builder) diff --git a/core/blockchain_snapshot_test.go b/core/blockchain_snapshot_test.go index dfa8ed65ec6d5..189eee59a009a 100644 --- a/core/blockchain_snapshot_test.go +++ b/core/blockchain_snapshot_test.go @@ -150,6 +150,7 @@ func (basic *snapshotTestBasic) verify(t *testing.T, chain *BlockChain, blocks [ } } +//nolint:unused func (basic *snapshotTestBasic) dump() string { buffer := new(strings.Builder) @@ -341,54 +342,6 @@ func (snaptest *setHeadSnapshotTest) test(t *testing.T) { snaptest.verify(t, newchain, blocks) } -// restartCrashSnapshotTest is the test type used to test this scenario: -// - have a complete snapshot -// - restart chain -// - insert more blocks with enabling the snapshot -// - commit the snapshot -// - crash -// - restart again -type restartCrashSnapshotTest struct { - snapshotTestBasic - newBlocks int -} - -func (snaptest *restartCrashSnapshotTest) test(t *testing.T) { - // It's hard to follow the test case, visualize the input - // log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true)))) - // fmt.Println(tt.dump()) - chain, blocks := snaptest.prepare(t) - - // Firstly, stop the chain properly, with all snapshot journal - // and state committed. - chain.Stop() - - newchain, err := NewBlockChain(snaptest.db, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil) - if err != nil { - t.Fatalf("Failed to recreate chain: %v", err) - } - newBlocks, _ := GenerateChain(params.TestChainConfig, blocks[len(blocks)-1], snaptest.engine, snaptest.gendb, snaptest.newBlocks, func(i int, b *BlockGen) {}) - newchain.InsertChain(newBlocks) - - // Commit the entire snapshot into the disk if requested. Note only - // (a) snapshot root and (b) snapshot generator will be committed, - // the diff journal is not. - newchain.Snapshots().Cap(newBlocks[len(newBlocks)-1].Root(), 0) - - // Simulate the blockchain crash - // Don't call chain.Stop here, so that no snapshot - // journal and latest state will be committed - - // Restart the chain after the crash - newchain, err = NewBlockChain(snaptest.db, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil) - if err != nil { - t.Fatalf("Failed to recreate chain: %v", err) - } - defer newchain.Stop() - - snaptest.verify(t, newchain, blocks) -} - // wipeCrashSnapshotTest is the test type used to test this scenario: // - have a complete snapshot // - restart, insert more blocks without enabling the snapshot @@ -431,7 +384,7 @@ func (snaptest *wipeCrashSnapshotTest) test(t *testing.T) { SnapshotLimit: 256, SnapshotWait: false, // Don't wait rebuild } - newchain, err = NewBlockChain(snaptest.db, config, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil) + _, err = NewBlockChain(snaptest.db, config, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to recreate chain: %v", err) } diff --git a/core/blockchain_test.go b/core/blockchain_test.go index b42f572b12900..44256f6187daa 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -503,7 +503,7 @@ func TestReorgShortBlocks(t *testing.T) { testReorgShort(t, true) } func testReorgShort(t *testing.T, full bool) { // Create a long easy chain vs. a short heavy one. Due to difficulty adjustment // we need a fairly long chain of blocks with different difficulties for a short - // one to become heavyer than a long one. The 96 is an empirical value. + // one to become heavier than a long one. The 96 is an empirical value. easy := make([]int64, 96) for i := 0; i < len(easy); i++ { easy[i] = 60 @@ -759,9 +759,9 @@ func TestFastVsFullChains(t *testing.T) { block.AddTx(tx) } } - // If the block number is a multiple of 5, add a few bonus uncles to the block - if i%5 == 5 { - block.AddUncle(&types.Header{ParentHash: block.PrevBlock(i - 1).Hash(), Number: big.NewInt(int64(i - 1))}) + // If the block number is a multiple of 5, add an uncle to the block + if i%5 == 4 { + block.AddUncle(&types.Header{ParentHash: block.PrevBlock(i - 2).Hash(), Number: big.NewInt(int64(i))}) } }) // Import the chain as an archive node for the comparison baseline @@ -1235,7 +1235,6 @@ func TestSideLogRebirth(t *testing.T) { chain, _ := GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), db, 2, func(i int, gen *BlockGen) { if i == 1 { gen.OffsetTime(-9) // higher block difficulty - } }) if _, err := blockchain.InsertChain(chain); err != nil { @@ -1364,7 +1363,6 @@ done: t.Errorf("unexpected event fired: %v", e) case <-time.After(250 * time.Millisecond): } - } // Tests if the canonical block can be fetched from the database during chain insertion. @@ -1943,8 +1941,8 @@ func testSideImport(t *testing.T, numCanonBlocksInSidechain, blocksBetweenCommon Alloc: GenesisAlloc{addr: {Balance: big.NewInt(math.MaxInt64)}}, BaseFee: big.NewInt(params.InitialBaseFee), } - signer = types.LatestSigner(gspec.Config) - genesis, _ = gspec.Commit(db) + signer = types.LatestSigner(gspec.Config) + genesis = gspec.MustCommit(db) ) // Generate and import the canonical chain diskdb := rawdb.NewMemoryDatabase() @@ -2576,6 +2574,7 @@ func TestTransactionIndices(t *testing.T) { t.Fatalf("failed to create temp freezer db: %v", err) } gspec.MustCommit(ancientDb) + l := l chain, err = NewBlockChain(ancientDb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, &l) if err != nil { t.Fatalf("failed to create tester chain: %v", err) @@ -2601,6 +2600,7 @@ func TestTransactionIndices(t *testing.T) { limit = []uint64{0, 64 /* drop stale */, 32 /* shorten history */, 64 /* extend history */, 0 /* restore all */} tails := []uint64{0, 67 /* 130 - 64 + 1 */, 100 /* 131 - 32 + 1 */, 69 /* 132 - 64 + 1 */, 0} for i, l := range limit { + l := l chain, err = NewBlockChain(ancientDb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, &l) if err != nil { t.Fatalf("failed to create tester chain: %v", err) @@ -2751,7 +2751,6 @@ func benchmarkLargeNumberOfValueToNonexisting(b *testing.B, numTxs, numBlocks in b.StopTimer() if got := chain.CurrentBlock().Transactions().Len(); got != numTxs*numBlocks { b.Fatalf("Transactions were not included, expected %d, got %d", numTxs*numBlocks, got) - } } } @@ -2762,7 +2761,7 @@ func BenchmarkBlockChain_1x1000ValueTransferToNonexisting(b *testing.B) { numBlocks = 1 ) recipientFn := func(nonce uint64) common.Address { - return common.BigToAddress(big.NewInt(0).SetUint64(1337 + nonce)) + return common.BigToAddress(new(big.Int).SetUint64(1337 + nonce)) } dataFn := func(nonce uint64) []byte { return nil @@ -2779,7 +2778,7 @@ func BenchmarkBlockChain_1x1000ValueTransferToExisting(b *testing.B) { b.ResetTimer() recipientFn := func(nonce uint64) common.Address { - return common.BigToAddress(big.NewInt(0).SetUint64(1337)) + return common.BigToAddress(new(big.Int).SetUint64(1337)) } dataFn := func(nonce uint64) []byte { return nil @@ -2796,7 +2795,7 @@ func BenchmarkBlockChain_1x1000Executions(b *testing.B) { b.ResetTimer() recipientFn := func(nonce uint64) common.Address { - return common.BigToAddress(big.NewInt(0).SetUint64(0xc0de)) + return common.BigToAddress(new(big.Int).SetUint64(0xc0de)) } dataFn := func(nonce uint64) []byte { return nil @@ -3520,7 +3519,6 @@ func TestEIP2718Transition(t *testing.T) { vm.GasQuickStep*2 + params.WarmStorageReadCostEIP2929 + params.ColdSloadCostEIP2929 if block.GasUsed() != expected { t.Fatalf("incorrect amount of gas spent: expected %d, got %d", expected, block.GasUsed()) - } } @@ -3758,3 +3756,112 @@ func TestSetCanonical(t *testing.T) { chain.SetCanonical(canon[TriesInMemory-1]) verify(canon[TriesInMemory-1]) } + +// TestCanonicalHashMarker tests all the canonical hash markers are updated/deleted +// correctly in case reorg is called. +func TestCanonicalHashMarker(t *testing.T) { + var cases = []struct { + forkA int + forkB int + }{ + // ForkA: 10 blocks + // ForkB: 1 blocks + // + // reorged: + // markers [2, 10] should be deleted + // markers [1] should be updated + {10, 1}, + + // ForkA: 10 blocks + // ForkB: 2 blocks + // + // reorged: + // markers [3, 10] should be deleted + // markers [1, 2] should be updated + {10, 2}, + + // ForkA: 10 blocks + // ForkB: 10 blocks + // + // reorged: + // markers [1, 10] should be updated + {10, 10}, + + // ForkA: 10 blocks + // ForkB: 11 blocks + // + // reorged: + // markers [1, 11] should be updated + {10, 11}, + } + for _, c := range cases { + var ( + db = rawdb.NewMemoryDatabase() + gspec = &Genesis{ + Config: params.TestChainConfig, + Alloc: GenesisAlloc{}, + BaseFee: big.NewInt(params.InitialBaseFee), + } + genesis = gspec.MustCommit(db) + engine = ethash.NewFaker() + ) + forkA, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, c.forkA, func(i int, gen *BlockGen) {}) + forkB, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, c.forkB, func(i int, gen *BlockGen) {}) + + // Initialize test chain + diskdb := rawdb.NewMemoryDatabase() + gspec.MustCommit(diskdb) + chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil, nil) + if err != nil { + t.Fatalf("failed to create tester chain: %v", err) + } + // Insert forkA and forkB, the canonical should on forkA still + if n, err := chain.InsertChain(forkA); err != nil { + t.Fatalf("block %d: failed to insert into chain: %v", n, err) + } + if n, err := chain.InsertChain(forkB); err != nil { + t.Fatalf("block %d: failed to insert into chain: %v", n, err) + } + + verify := func(head *types.Block) { + if chain.CurrentBlock().Hash() != head.Hash() { + t.Fatalf("Unexpected block hash, want %x, got %x", head.Hash(), chain.CurrentBlock().Hash()) + } + if chain.CurrentFastBlock().Hash() != head.Hash() { + t.Fatalf("Unexpected fast block hash, want %x, got %x", head.Hash(), chain.CurrentFastBlock().Hash()) + } + if chain.CurrentHeader().Hash() != head.Hash() { + t.Fatalf("Unexpected head header, want %x, got %x", head.Hash(), chain.CurrentHeader().Hash()) + } + if !chain.HasState(head.Root()) { + t.Fatalf("Lost block state %v %x", head.Number(), head.Hash()) + } + } + + // Switch canonical chain to forkB if necessary + if len(forkA) < len(forkB) { + verify(forkB[len(forkB)-1]) + } else { + verify(forkA[len(forkA)-1]) + chain.SetCanonical(forkB[len(forkB)-1]) + verify(forkB[len(forkB)-1]) + } + + // Ensure all hash markers are updated correctly + for i := 0; i < len(forkB); i++ { + block := forkB[i] + hash := chain.GetCanonicalHash(block.NumberU64()) + if hash != block.Hash() { + t.Fatalf("Unexpected canonical hash %d", block.NumberU64()) + } + } + if c.forkA > c.forkB { + for i := uint64(c.forkB) + 1; i <= uint64(c.forkA); i++ { + hash := chain.GetCanonicalHash(i) + if hash != (common.Hash{}) { + t.Fatalf("Unexpected canonical hash %d", i) + } + } + } + } +} diff --git a/core/bloom_indexer.go b/core/bloom_indexer.go index 856746a1c0883..68a35d811e41e 100644 --- a/core/bloom_indexer.go +++ b/core/bloom_indexer.go @@ -75,7 +75,7 @@ func (b *BloomIndexer) Process(ctx context.Context, header *types.Header) error // Commit implements core.ChainIndexerBackend, finalizing the bloom section and // writing it out into the database. func (b *BloomIndexer) Commit() error { - batch := b.db.NewBatch() + batch := b.db.NewBatchWithSize((int(b.size) / 8) * types.BloomBitLength) for i := 0; i < types.BloomBitLength; i++ { bits, err := b.gen.Bitset(uint(i)) if err != nil { diff --git a/core/bloombits/matcher_test.go b/core/bloombits/matcher_test.go index 923579221f516..93d4632b8587b 100644 --- a/core/bloombits/matcher_test.go +++ b/core/bloombits/matcher_test.go @@ -124,13 +124,13 @@ func makeRandomIndexes(lengths []int, max int) [][]bloomIndexes { // testMatcherDiffBatches runs the given matches test in single-delivery and also // in batches delivery mode, verifying that all kinds of deliveries are handled -// correctly withn. +// correctly within. func testMatcherDiffBatches(t *testing.T, filter [][]bloomIndexes, start, blocks uint64, intermittent bool, retrievals uint32) { singleton := testMatcher(t, filter, start, blocks, intermittent, retrievals, 1) batched := testMatcher(t, filter, start, blocks, intermittent, retrievals, 16) if singleton != batched { - t.Errorf("filter = %v blocks = %v intermittent = %v: request count mismatch, %v in signleton vs. %v in batched mode", filter, blocks, intermittent, singleton, batched) + t.Errorf("filter = %v blocks = %v intermittent = %v: request count mismatch, %v in singleton vs. %v in batched mode", filter, blocks, intermittent, singleton, batched) } } diff --git a/core/evm.go b/core/evm.go index 536ac673e6a62..e929da25eaeea 100644 --- a/core/evm.go +++ b/core/evm.go @@ -31,7 +31,7 @@ type ChainContext interface { // Engine retrieves the chain's consensus engine. Engine() consensus.Engine - // GetHeader returns the hash corresponding to their hash. + // GetHeader returns the header corresponding to the hash/number argument pair. GetHeader(common.Hash, uint64) *types.Header } @@ -84,6 +84,11 @@ func GetHashFn(ref *types.Header, chain ChainContext) func(n uint64) common.Hash var cache []common.Hash return func(n uint64) common.Hash { + if ref.Number.Uint64() <= n { + // This situation can happen if we're doing tracing and using + // block overrides. + return common.Hash{} + } // If there's no hash cache yet, make one if len(cache) == 0 { cache = append(cache, ref.ParentHash) diff --git a/core/forkid/forkid_test.go b/core/forkid/forkid_test.go index b0ee59b9eb7b2..2a0fb167d5166 100644 --- a/core/forkid/forkid_test.go +++ b/core/forkid/forkid_test.go @@ -31,7 +31,7 @@ import ( // the correct fork ID. func TestCreation(t *testing.T) { mergeConfig := *params.MainnetChainConfig - mergeConfig.MergeForkBlock = big.NewInt(15000000) + mergeConfig.MergeNetsplitBlock = big.NewInt(18000000) type testcase struct { head uint64 want ID @@ -68,8 +68,10 @@ func TestCreation(t *testing.T) { {12964999, ID{Hash: checksumToBytes(0x0eb440f6), Next: 12965000}}, // Last Berlin block {12965000, ID{Hash: checksumToBytes(0xb715077d), Next: 13773000}}, // First London block {13772999, ID{Hash: checksumToBytes(0xb715077d), Next: 13773000}}, // Last London block - {13773000, ID{Hash: checksumToBytes(0x20c327fc), Next: 0}}, // First Arrow Glacier block - {20000000, ID{Hash: checksumToBytes(0x20c327fc), Next: 0}}, // Future Arrow Glacier block + {13773000, ID{Hash: checksumToBytes(0x20c327fc), Next: 15050000}}, // First Arrow Glacier block + {15049999, ID{Hash: checksumToBytes(0x20c327fc), Next: 15050000}}, // Last Arrow Glacier block + {15050000, ID{Hash: checksumToBytes(0xf0afd0e3), Next: 0}}, // First Gray Glacier block + {20000000, ID{Hash: checksumToBytes(0xf0afd0e3), Next: 0}}, // Future Gray Glacier block }, }, // Ropsten test cases @@ -136,6 +138,16 @@ func TestCreation(t *testing.T) { {6000000, ID{Hash: checksumToBytes(0xB8C6299D), Next: 0}}, // Future London block }, }, + // Sepolia test cases + { + params.SepoliaChainConfig, + params.SepoliaGenesisHash, + []testcase{ + {0, ID{Hash: checksumToBytes(0xfe3366e7), Next: 1735371}}, // Unsynced, last Frontier, Homestead, Tangerine, Spurious, Byzantium, Constantinople, Petersburg, Istanbul, Berlin and first London block + {1735370, ID{Hash: checksumToBytes(0xfe3366e7), Next: 1735371}}, // Last London block + {1735371, ID{Hash: checksumToBytes(0xb96cbd13), Next: 0}}, // First MergeNetsplit block + }, + }, // Merge test cases { &mergeConfig, @@ -163,9 +175,11 @@ func TestCreation(t *testing.T) { {12964999, ID{Hash: checksumToBytes(0x0eb440f6), Next: 12965000}}, // Last Berlin block {12965000, ID{Hash: checksumToBytes(0xb715077d), Next: 13773000}}, // First London block {13772999, ID{Hash: checksumToBytes(0xb715077d), Next: 13773000}}, // Last London block - {13773000, ID{Hash: checksumToBytes(0x20c327fc), Next: 15000000}}, // First Arrow Glacier block - {15000000, ID{Hash: checksumToBytes(0xe3abe201), Next: 0}}, // First Merge Start block - {20000000, ID{Hash: checksumToBytes(0xe3abe201), Next: 0}}, // Future Merge Start block + {13773000, ID{Hash: checksumToBytes(0x20c327fc), Next: 15050000}}, // First Arrow Glacier block + {15049999, ID{Hash: checksumToBytes(0x20c327fc), Next: 15050000}}, // Last Arrow Glacier block + {15050000, ID{Hash: checksumToBytes(0xf0afd0e3), Next: 18000000}}, // First Gray Glacier block + {18000000, ID{Hash: checksumToBytes(0x4fb8a872), Next: 0}}, // First Merge Start block + {20000000, ID{Hash: checksumToBytes(0x4fb8a872), Next: 0}}, // Future Merge Start block }, }, } @@ -242,11 +256,11 @@ func TestValidation(t *testing.T) { // Local is mainnet Petersburg, remote is Rinkeby Petersburg. {7987396, ID{Hash: checksumToBytes(0xafec6b27), Next: 0}, ErrLocalIncompatibleOrStale}, - // Local is mainnet Arrow Glacier, far in the future. Remote announces Gopherium (non existing fork) + // Local is mainnet Gray Glacier, far in the future. Remote announces Gopherium (non existing fork) // at some future block 88888888, for itself, but past block for local. Local is incompatible. // // This case detects non-upgraded nodes with majority hash power (typical Ropsten mess). - {88888888, ID{Hash: checksumToBytes(0x20c327fc), Next: 88888888}, ErrLocalIncompatibleOrStale}, + {88888888, ID{Hash: checksumToBytes(0xf0afd0e3), Next: 88888888}, ErrLocalIncompatibleOrStale}, // Local is mainnet Byzantium. Remote is also in Byzantium, but announces Gopherium (non existing // fork) at block 7279999, before Petersburg. Local is incompatible. diff --git a/core/genesis.go b/core/genesis.go index 64ee99c5443de..c3b5d0b570945 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -80,10 +80,12 @@ func (ga *GenesisAlloc) UnmarshalJSON(data []byte) error { return nil } -// flush adds allocated genesis accounts into a fresh new statedb and -// commit the state changes into the given database handler. -func (ga *GenesisAlloc) flush(db ethdb.Database) (common.Hash, error) { - statedb, err := state.New(common.Hash{}, state.NewDatabase(db), nil) +// deriveHash computes the state root according to the genesis specification. +func (ga *GenesisAlloc) deriveHash() (common.Hash, error) { + // Create an ephemeral in-memory database for computing hash, + // all the derived states will be discarded to not pollute disk. + db := state.NewDatabase(rawdb.NewMemoryDatabase()) + statedb, err := state.New(common.Hash{}, db, nil) if err != nil { return common.Hash{}, err } @@ -95,25 +97,39 @@ func (ga *GenesisAlloc) flush(db ethdb.Database) (common.Hash, error) { statedb.SetState(addr, key, value) } } + return statedb.Commit(false) +} + +// flush is very similar with deriveHash, but the main difference is +// all the generated states will be persisted into the given database. +// Also, the genesis state specification will be flushed as well. +func (ga *GenesisAlloc) flush(db ethdb.Database) error { + statedb, err := state.New(common.Hash{}, state.NewDatabaseWithConfig(db, &trie.Config{Preimages: true}), nil) + if err != nil { + return err + } + for addr, account := range *ga { + statedb.AddBalance(addr, account.Balance) + statedb.SetCode(addr, account.Code) + statedb.SetNonce(addr, account.Nonce) + for key, value := range account.Storage { + statedb.SetState(addr, key, value) + } + } root, err := statedb.Commit(false) if err != nil { - return common.Hash{}, err + return err } err = statedb.Database().TrieDB().Commit(root, true, nil) if err != nil { - return common.Hash{}, err + return err } - return root, nil -} - -// write writes the json marshaled genesis state into database -// with the given block hash as the unique identifier. -func (ga *GenesisAlloc) write(db ethdb.KeyValueWriter, hash common.Hash) error { + // Marshal the genesis state specification and persist. blob, err := json.Marshal(ga) if err != nil { return err } - rawdb.WriteGenesisState(db, hash, blob) + rawdb.WriteGenesisStateSpec(db, root, blob) return nil } @@ -121,7 +137,7 @@ func (ga *GenesisAlloc) write(db ethdb.KeyValueWriter, hash common.Hash) error { // hash and commits them into the given database handler. func CommitGenesisState(db ethdb.Database, hash common.Hash) error { var alloc GenesisAlloc - blob := rawdb.ReadGenesisState(db, hash) + blob := rawdb.ReadGenesisStateSpec(db, hash) if len(blob) != 0 { if err := alloc.UnmarshalJSON(blob); err != nil { return err @@ -151,8 +167,7 @@ func CommitGenesisState(db ethdb.Database, hash common.Hash) error { return errors.New("not found") } } - _, err := alloc.flush(db) - return err + return alloc.flush(db) } // GenesisAccount is an account in the state of the genesis block. @@ -233,10 +248,22 @@ func SetupGenesisBlock(db ethdb.Database, genesis *Genesis) (*params.ChainConfig return SetupGenesisBlockWithOverride(db, genesis, nil, nil) } -func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, overrideArrowGlacier, overrideTerminalTotalDifficulty *big.Int) (*params.ChainConfig, common.Hash, error) { +func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, overrideTerminalTotalDifficulty *big.Int, overrideTerminalTotalDifficultyPassed *bool) (*params.ChainConfig, common.Hash, error) { if genesis != nil && genesis.Config == nil { return params.AllEthashProtocolChanges, common.Hash{}, errGenesisNoConfig } + + applyOverrides := func(config *params.ChainConfig) { + if config != nil { + if overrideTerminalTotalDifficulty != nil { + config.TerminalTotalDifficulty = overrideTerminalTotalDifficulty + } + if overrideTerminalTotalDifficultyPassed != nil { + config.TerminalTotalDifficultyPassed = *overrideTerminalTotalDifficultyPassed + } + } + } + // Just commit the new block if there is no stored genesis block. stored := rawdb.ReadCanonicalHash(db, 0) if (stored == common.Hash{}) { @@ -250,6 +277,7 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, override if err != nil { return genesis.Config, common.Hash{}, err } + applyOverrides(genesis.Config) return genesis.Config, block.Hash(), nil } // We have the genesis block in database(perhaps in ancient database) @@ -260,7 +288,7 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, override genesis = DefaultGenesisBlock() } // Ensure the stored genesis matches with the given one. - hash := genesis.ToBlock(nil).Hash() + hash := genesis.ToBlock().Hash() if hash != stored { return genesis.Config, hash, &GenesisMismatchError{stored, hash} } @@ -268,23 +296,19 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, override if err != nil { return genesis.Config, hash, err } + applyOverrides(genesis.Config) return genesis.Config, block.Hash(), nil } // Check whether the genesis block is already written. if genesis != nil { - hash := genesis.ToBlock(nil).Hash() + hash := genesis.ToBlock().Hash() if hash != stored { return genesis.Config, hash, &GenesisMismatchError{stored, hash} } } // Get the existing chain configuration. newcfg := genesis.configOrDefault(stored) - if overrideArrowGlacier != nil { - newcfg.ArrowGlacierBlock = overrideArrowGlacier - } - if overrideTerminalTotalDifficulty != nil { - newcfg.TerminalTotalDifficulty = overrideTerminalTotalDifficulty - } + applyOverrides(newcfg) if err := newcfg.CheckConfigForkOrder(); err != nil { return newcfg, common.Hash{}, err } @@ -301,12 +325,7 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, override // apply the overrides. if genesis == nil && stored != params.MainnetGenesisHash { newcfg = storedcfg - if overrideArrowGlacier != nil { - newcfg.ArrowGlacierBlock = overrideArrowGlacier - } - if overrideTerminalTotalDifficulty != nil { - newcfg.TerminalTotalDifficulty = overrideTerminalTotalDifficulty - } + applyOverrides(newcfg) } // Check config compatibility and write the config. Compatibility errors // are returned to the caller unless we're already at block zero. @@ -343,13 +362,9 @@ func (g *Genesis) configOrDefault(ghash common.Hash) *params.ChainConfig { } } -// ToBlock creates the genesis block and writes state of a genesis specification -// to the given database (or discards it if nil). -func (g *Genesis) ToBlock(db ethdb.Database) *types.Block { - if db == nil { - db = rawdb.NewMemoryDatabase() - } - root, err := g.Alloc.flush(db) +// ToBlock returns the genesis block according to genesis specification. +func (g *Genesis) ToBlock() *types.Block { + root, err := g.Alloc.deriveHash() if err != nil { panic(err) } @@ -386,7 +401,7 @@ func (g *Genesis) ToBlock(db ethdb.Database) *types.Block { // Commit writes the block and state of a genesis specification to the database. // The block is committed as the canonical head block. func (g *Genesis) Commit(db ethdb.Database) (*types.Block, error) { - block := g.ToBlock(db) + block := g.ToBlock() if block.Number().Sign() != 0 { return nil, errors.New("can't commit genesis block with number > 0") } @@ -400,7 +415,10 @@ func (g *Genesis) Commit(db ethdb.Database) (*types.Block, error) { if config.Clique != nil && len(block.Extra()) < 32+crypto.SignatureLength { return nil, errors.New("can't start clique chain without signers") } - if err := g.Alloc.write(db, block.Hash()); err != nil { + // All the checks has passed, flush the states derived from the genesis + // specification as well as the specification itself into the provided + // database. + if err := g.Alloc.flush(db); err != nil { return nil, err } rawdb.WriteTd(db, block.Hash(), block.NumberU64(), block.Difficulty()) @@ -424,15 +442,6 @@ func (g *Genesis) MustCommit(db ethdb.Database) *types.Block { return block } -// GenesisBlockForTesting creates and writes a block in which addr has the given wei balance. -func GenesisBlockForTesting(db ethdb.Database, addr common.Address, balance *big.Int) *types.Block { - g := Genesis{ - Alloc: GenesisAlloc{addr: {Balance: balance}}, - BaseFee: big.NewInt(params.InitialBaseFee), - } - return g.MustCommit(db) -} - // DefaultGenesisBlock returns the Ethereum main net genesis block. func DefaultGenesisBlock() *Genesis { return &Genesis{ @@ -494,6 +503,7 @@ func DefaultSepoliaGenesisBlock() *Genesis { } } +// DefaultKilnGenesisBlock returns the kiln network genesis block. func DefaultKilnGenesisBlock() *Genesis { g := new(Genesis) reader := strings.NewReader(KilnAllocData) diff --git a/core/genesis_alloc.go b/core/genesis_alloc.go index 041c55424238d..16df390575c21 100644 --- a/core/genesis_alloc.go +++ b/core/genesis_alloc.go @@ -25,7 +25,6 @@ const mainnetAllocData = "\xfa\x04]X\u0793\r\x83b\x011\x8e\u0189\x9agT\x06\x908' const ropstenAllocData = "\xf9\x03\xa4\u0080\x01\xc2\x01\x01\xc2\x02\x01\xc2\x03\x01\xc2\x04\x01\xc2\x05\x01\xc2\x06\x01\xc2\a\x01\xc2\b\x01\xc2\t\x01\xc2\n\x80\xc2\v\x80\xc2\f\x80\xc2\r\x80\xc2\x0e\x80\xc2\x0f\x80\xc2\x10\x80\xc2\x11\x80\xc2\x12\x80\xc2\x13\x80\xc2\x14\x80\xc2\x15\x80\xc2\x16\x80\xc2\x17\x80\xc2\x18\x80\xc2\x19\x80\xc2\x1a\x80\xc2\x1b\x80\xc2\x1c\x80\xc2\x1d\x80\xc2\x1e\x80\xc2\x1f\x80\xc2 \x80\xc2!\x80\xc2\"\x80\xc2#\x80\xc2$\x80\xc2%\x80\xc2&\x80\xc2'\x80\xc2(\x80\xc2)\x80\xc2*\x80\xc2+\x80\xc2,\x80\xc2-\x80\xc2.\x80\xc2/\x80\xc20\x80\xc21\x80\xc22\x80\xc23\x80\xc24\x80\xc25\x80\xc26\x80\xc27\x80\xc28\x80\xc29\x80\xc2:\x80\xc2;\x80\xc2<\x80\xc2=\x80\xc2>\x80\xc2?\x80\xc2@\x80\xc2A\x80\xc2B\x80\xc2C\x80\xc2D\x80\xc2E\x80\xc2F\x80\xc2G\x80\xc2H\x80\xc2I\x80\xc2J\x80\xc2K\x80\xc2L\x80\xc2M\x80\xc2N\x80\xc2O\x80\xc2P\x80\xc2Q\x80\xc2R\x80\xc2S\x80\xc2T\x80\xc2U\x80\xc2V\x80\xc2W\x80\xc2X\x80\xc2Y\x80\xc2Z\x80\xc2[\x80\xc2\\\x80\xc2]\x80\xc2^\x80\xc2_\x80\xc2`\x80\xc2a\x80\xc2b\x80\xc2c\x80\xc2d\x80\xc2e\x80\xc2f\x80\xc2g\x80\xc2h\x80\xc2i\x80\xc2j\x80\xc2k\x80\xc2l\x80\xc2m\x80\xc2n\x80\xc2o\x80\xc2p\x80\xc2q\x80\xc2r\x80\xc2s\x80\xc2t\x80\xc2u\x80\xc2v\x80\xc2w\x80\xc2x\x80\xc2y\x80\xc2z\x80\xc2{\x80\xc2|\x80\xc2}\x80\xc2~\x80\xc2\u007f\x80\u00c1\x80\x80\u00c1\x81\x80\u00c1\x82\x80\u00c1\x83\x80\u00c1\x84\x80\u00c1\x85\x80\u00c1\x86\x80\u00c1\x87\x80\u00c1\x88\x80\u00c1\x89\x80\u00c1\x8a\x80\u00c1\x8b\x80\u00c1\x8c\x80\u00c1\x8d\x80\u00c1\x8e\x80\u00c1\x8f\x80\u00c1\x90\x80\u00c1\x91\x80\u00c1\x92\x80\u00c1\x93\x80\u00c1\x94\x80\u00c1\x95\x80\u00c1\x96\x80\u00c1\x97\x80\u00c1\x98\x80\u00c1\x99\x80\u00c1\x9a\x80\u00c1\x9b\x80\u00c1\x9c\x80\u00c1\x9d\x80\u00c1\x9e\x80\u00c1\x9f\x80\u00c1\xa0\x80\u00c1\xa1\x80\u00c1\xa2\x80\u00c1\xa3\x80\u00c1\xa4\x80\u00c1\xa5\x80\u00c1\xa6\x80\u00c1\xa7\x80\u00c1\xa8\x80\u00c1\xa9\x80\u00c1\xaa\x80\u00c1\xab\x80\u00c1\xac\x80\u00c1\xad\x80\u00c1\xae\x80\u00c1\xaf\x80\u00c1\xb0\x80\u00c1\xb1\x80\u00c1\xb2\x80\u00c1\xb3\x80\u00c1\xb4\x80\u00c1\xb5\x80\u00c1\xb6\x80\u00c1\xb7\x80\u00c1\xb8\x80\u00c1\xb9\x80\u00c1\xba\x80\u00c1\xbb\x80\u00c1\xbc\x80\u00c1\xbd\x80\u00c1\xbe\x80\u00c1\xbf\x80\u00c1\xc0\x80\u00c1\xc1\x80\u00c1\u0080\u00c1\u00c0\u00c1\u0100\u00c1\u0140\u00c1\u0180\u00c1\u01c0\u00c1\u0200\u00c1\u0240\u00c1\u0280\u00c1\u02c0\u00c1\u0300\u00c1\u0340\u00c1\u0380\u00c1\u03c0\u00c1\u0400\u00c1\u0440\u00c1\u0480\u00c1\u04c0\u00c1\u0500\u00c1\u0540\u00c1\u0580\u00c1\u05c0\u00c1\u0600\u00c1\u0640\u00c1\u0680\u00c1\u06c0\u00c1\u0700\u00c1\u0740\u00c1\u0780\u00c1\u07c0\u00c1\xe0\x80\u00c1\xe1\x80\u00c1\xe2\x80\u00c1\xe3\x80\u00c1\xe4\x80\u00c1\xe5\x80\u00c1\xe6\x80\u00c1\xe7\x80\u00c1\xe8\x80\u00c1\xe9\x80\u00c1\xea\x80\u00c1\xeb\x80\u00c1\xec\x80\u00c1\xed\x80\u00c1\xee\x80\u00c1\xef\x80\u00c1\xf0\x80\u00c1\xf1\x80\u00c1\xf2\x80\u00c1\xf3\x80\u00c1\xf4\x80\u00c1\xf5\x80\u00c1\xf6\x80\u00c1\xf7\x80\u00c1\xf8\x80\u00c1\xf9\x80\u00c1\xfa\x80\u00c1\xfb\x80\u00c1\xfc\x80\u00c1\xfd\x80\u00c1\xfe\x80\u00c1\xff\x80\u3507KT\xa8\xbd\x15)f\xd6?pk\xae\x1f\xfe\xb0A\x19!\xe5\x8d\f\x9f,\x9c\xd0Ft\xed\xea@\x00\x00\x00" const rinkebyAllocData = "\xf9\x03\xb7\u0080\x01\xc2\x01\x01\xc2\x02\x01\xc2\x03\x01\xc2\x04\x01\xc2\x05\x01\xc2\x06\x01\xc2\a\x01\xc2\b\x01\xc2\t\x01\xc2\n\x01\xc2\v\x01\xc2\f\x01\xc2\r\x01\xc2\x0e\x01\xc2\x0f\x01\xc2\x10\x01\xc2\x11\x01\xc2\x12\x01\xc2\x13\x01\xc2\x14\x01\xc2\x15\x01\xc2\x16\x01\xc2\x17\x01\xc2\x18\x01\xc2\x19\x01\xc2\x1a\x01\xc2\x1b\x01\xc2\x1c\x01\xc2\x1d\x01\xc2\x1e\x01\xc2\x1f\x01\xc2 \x01\xc2!\x01\xc2\"\x01\xc2#\x01\xc2$\x01\xc2%\x01\xc2&\x01\xc2'\x01\xc2(\x01\xc2)\x01\xc2*\x01\xc2+\x01\xc2,\x01\xc2-\x01\xc2.\x01\xc2/\x01\xc20\x01\xc21\x01\xc22\x01\xc23\x01\xc24\x01\xc25\x01\xc26\x01\xc27\x01\xc28\x01\xc29\x01\xc2:\x01\xc2;\x01\xc2<\x01\xc2=\x01\xc2>\x01\xc2?\x01\xc2@\x01\xc2A\x01\xc2B\x01\xc2C\x01\xc2D\x01\xc2E\x01\xc2F\x01\xc2G\x01\xc2H\x01\xc2I\x01\xc2J\x01\xc2K\x01\xc2L\x01\xc2M\x01\xc2N\x01\xc2O\x01\xc2P\x01\xc2Q\x01\xc2R\x01\xc2S\x01\xc2T\x01\xc2U\x01\xc2V\x01\xc2W\x01\xc2X\x01\xc2Y\x01\xc2Z\x01\xc2[\x01\xc2\\\x01\xc2]\x01\xc2^\x01\xc2_\x01\xc2`\x01\xc2a\x01\xc2b\x01\xc2c\x01\xc2d\x01\xc2e\x01\xc2f\x01\xc2g\x01\xc2h\x01\xc2i\x01\xc2j\x01\xc2k\x01\xc2l\x01\xc2m\x01\xc2n\x01\xc2o\x01\xc2p\x01\xc2q\x01\xc2r\x01\xc2s\x01\xc2t\x01\xc2u\x01\xc2v\x01\xc2w\x01\xc2x\x01\xc2y\x01\xc2z\x01\xc2{\x01\xc2|\x01\xc2}\x01\xc2~\x01\xc2\u007f\x01\u00c1\x80\x01\u00c1\x81\x01\u00c1\x82\x01\u00c1\x83\x01\u00c1\x84\x01\u00c1\x85\x01\u00c1\x86\x01\u00c1\x87\x01\u00c1\x88\x01\u00c1\x89\x01\u00c1\x8a\x01\u00c1\x8b\x01\u00c1\x8c\x01\u00c1\x8d\x01\u00c1\x8e\x01\u00c1\x8f\x01\u00c1\x90\x01\u00c1\x91\x01\u00c1\x92\x01\u00c1\x93\x01\u00c1\x94\x01\u00c1\x95\x01\u00c1\x96\x01\u00c1\x97\x01\u00c1\x98\x01\u00c1\x99\x01\u00c1\x9a\x01\u00c1\x9b\x01\u00c1\x9c\x01\u00c1\x9d\x01\u00c1\x9e\x01\u00c1\x9f\x01\u00c1\xa0\x01\u00c1\xa1\x01\u00c1\xa2\x01\u00c1\xa3\x01\u00c1\xa4\x01\u00c1\xa5\x01\u00c1\xa6\x01\u00c1\xa7\x01\u00c1\xa8\x01\u00c1\xa9\x01\u00c1\xaa\x01\u00c1\xab\x01\u00c1\xac\x01\u00c1\xad\x01\u00c1\xae\x01\u00c1\xaf\x01\u00c1\xb0\x01\u00c1\xb1\x01\u00c1\xb2\x01\u00c1\xb3\x01\u00c1\xb4\x01\u00c1\xb5\x01\u00c1\xb6\x01\u00c1\xb7\x01\u00c1\xb8\x01\u00c1\xb9\x01\u00c1\xba\x01\u00c1\xbb\x01\u00c1\xbc\x01\u00c1\xbd\x01\u00c1\xbe\x01\u00c1\xbf\x01\u00c1\xc0\x01\u00c1\xc1\x01\u00c1\xc2\x01\u00c1\xc3\x01\u00c1\xc4\x01\u00c1\xc5\x01\u00c1\xc6\x01\u00c1\xc7\x01\u00c1\xc8\x01\u00c1\xc9\x01\u00c1\xca\x01\u00c1\xcb\x01\u00c1\xcc\x01\u00c1\xcd\x01\u00c1\xce\x01\u00c1\xcf\x01\u00c1\xd0\x01\u00c1\xd1\x01\u00c1\xd2\x01\u00c1\xd3\x01\u00c1\xd4\x01\u00c1\xd5\x01\u00c1\xd6\x01\u00c1\xd7\x01\u00c1\xd8\x01\u00c1\xd9\x01\u00c1\xda\x01\u00c1\xdb\x01\u00c1\xdc\x01\u00c1\xdd\x01\u00c1\xde\x01\u00c1\xdf\x01\u00c1\xe0\x01\u00c1\xe1\x01\u00c1\xe2\x01\u00c1\xe3\x01\u00c1\xe4\x01\u00c1\xe5\x01\u00c1\xe6\x01\u00c1\xe7\x01\u00c1\xe8\x01\u00c1\xe9\x01\u00c1\xea\x01\u00c1\xeb\x01\u00c1\xec\x01\u00c1\xed\x01\u00c1\xee\x01\u00c1\xef\x01\u00c1\xf0\x01\u00c1\xf1\x01\u00c1\xf2\x01\u00c1\xf3\x01\u00c1\xf4\x01\u00c1\xf5\x01\u00c1\xf6\x01\u00c1\xf7\x01\u00c1\xf8\x01\u00c1\xf9\x01\u00c1\xfa\x01\u00c1\xfb\x01\u00c1\xfc\x01\u00c1\xfd\x01\u00c1\xfe\x01\u00c1\xff\x01\xf6\x941\xb9\x8d\x14\x00{\xde\xe67)\x80\x86\x98\x8a\v\xbd1\x18E#\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" const goerliAllocData = "\xf9\x04\x06\u0080\x01\xc2\x01\x01\xc2\x02\x01\xc2\x03\x01\xc2\x04\x01\xc2\x05\x01\xc2\x06\x01\xc2\a\x01\xc2\b\x01\xc2\t\x01\xc2\n\x01\xc2\v\x01\xc2\f\x01\xc2\r\x01\xc2\x0e\x01\xc2\x0f\x01\xc2\x10\x01\xc2\x11\x01\xc2\x12\x01\xc2\x13\x01\xc2\x14\x01\xc2\x15\x01\xc2\x16\x01\xc2\x17\x01\xc2\x18\x01\xc2\x19\x01\xc2\x1a\x01\xc2\x1b\x01\xc2\x1c\x01\xc2\x1d\x01\xc2\x1e\x01\xc2\x1f\x01\xc2 \x01\xc2!\x01\xc2\"\x01\xc2#\x01\xc2$\x01\xc2%\x01\xc2&\x01\xc2'\x01\xc2(\x01\xc2)\x01\xc2*\x01\xc2+\x01\xc2,\x01\xc2-\x01\xc2.\x01\xc2/\x01\xc20\x01\xc21\x01\xc22\x01\xc23\x01\xc24\x01\xc25\x01\xc26\x01\xc27\x01\xc28\x01\xc29\x01\xc2:\x01\xc2;\x01\xc2<\x01\xc2=\x01\xc2>\x01\xc2?\x01\xc2@\x01\xc2A\x01\xc2B\x01\xc2C\x01\xc2D\x01\xc2E\x01\xc2F\x01\xc2G\x01\xc2H\x01\xc2I\x01\xc2J\x01\xc2K\x01\xc2L\x01\xc2M\x01\xc2N\x01\xc2O\x01\xc2P\x01\xc2Q\x01\xc2R\x01\xc2S\x01\xc2T\x01\xc2U\x01\xc2V\x01\xc2W\x01\xc2X\x01\xc2Y\x01\xc2Z\x01\xc2[\x01\xc2\\\x01\xc2]\x01\xc2^\x01\xc2_\x01\xc2`\x01\xc2a\x01\xc2b\x01\xc2c\x01\xc2d\x01\xc2e\x01\xc2f\x01\xc2g\x01\xc2h\x01\xc2i\x01\xc2j\x01\xc2k\x01\xc2l\x01\xc2m\x01\xc2n\x01\xc2o\x01\xc2p\x01\xc2q\x01\xc2r\x01\xc2s\x01\xc2t\x01\xc2u\x01\xc2v\x01\xc2w\x01\xc2x\x01\xc2y\x01\xc2z\x01\xc2{\x01\xc2|\x01\xc2}\x01\xc2~\x01\xc2\u007f\x01\u00c1\x80\x01\u00c1\x81\x01\u00c1\x82\x01\u00c1\x83\x01\u00c1\x84\x01\u00c1\x85\x01\u00c1\x86\x01\u00c1\x87\x01\u00c1\x88\x01\u00c1\x89\x01\u00c1\x8a\x01\u00c1\x8b\x01\u00c1\x8c\x01\u00c1\x8d\x01\u00c1\x8e\x01\u00c1\x8f\x01\u00c1\x90\x01\u00c1\x91\x01\u00c1\x92\x01\u00c1\x93\x01\u00c1\x94\x01\u00c1\x95\x01\u00c1\x96\x01\u00c1\x97\x01\u00c1\x98\x01\u00c1\x99\x01\u00c1\x9a\x01\u00c1\x9b\x01\u00c1\x9c\x01\u00c1\x9d\x01\u00c1\x9e\x01\u00c1\x9f\x01\u00c1\xa0\x01\u00c1\xa1\x01\u00c1\xa2\x01\u00c1\xa3\x01\u00c1\xa4\x01\u00c1\xa5\x01\u00c1\xa6\x01\u00c1\xa7\x01\u00c1\xa8\x01\u00c1\xa9\x01\u00c1\xaa\x01\u00c1\xab\x01\u00c1\xac\x01\u00c1\xad\x01\u00c1\xae\x01\u00c1\xaf\x01\u00c1\xb0\x01\u00c1\xb1\x01\u00c1\xb2\x01\u00c1\xb3\x01\u00c1\xb4\x01\u00c1\xb5\x01\u00c1\xb6\x01\u00c1\xb7\x01\u00c1\xb8\x01\u00c1\xb9\x01\u00c1\xba\x01\u00c1\xbb\x01\u00c1\xbc\x01\u00c1\xbd\x01\u00c1\xbe\x01\u00c1\xbf\x01\u00c1\xc0\x01\u00c1\xc1\x01\u00c1\xc2\x01\u00c1\xc3\x01\u00c1\xc4\x01\u00c1\xc5\x01\u00c1\xc6\x01\u00c1\xc7\x01\u00c1\xc8\x01\u00c1\xc9\x01\u00c1\xca\x01\u00c1\xcb\x01\u00c1\xcc\x01\u00c1\xcd\x01\u00c1\xce\x01\u00c1\xcf\x01\u00c1\xd0\x01\u00c1\xd1\x01\u00c1\xd2\x01\u00c1\xd3\x01\u00c1\xd4\x01\u00c1\xd5\x01\u00c1\xd6\x01\u00c1\xd7\x01\u00c1\xd8\x01\u00c1\xd9\x01\u00c1\xda\x01\u00c1\xdb\x01\u00c1\xdc\x01\u00c1\xdd\x01\u00c1\xde\x01\u00c1\xdf\x01\u00c1\xe0\x01\u00c1\xe1\x01\u00c1\xe2\x01\u00c1\xe3\x01\u00c1\xe4\x01\u00c1\xe5\x01\u00c1\xe6\x01\u00c1\xe7\x01\u00c1\xe8\x01\u00c1\xe9\x01\u00c1\xea\x01\u00c1\xeb\x01\u00c1\xec\x01\u00c1\xed\x01\u00c1\xee\x01\u00c1\xef\x01\u00c1\xf0\x01\u00c1\xf1\x01\u00c1\xf2\x01\u00c1\xf3\x01\u00c1\xf4\x01\u00c1\xf5\x01\u00c1\xf6\x01\u00c1\xf7\x01\u00c1\xf8\x01\u00c1\xf9\x01\u00c1\xfa\x01\u00c1\xfb\x01\u00c1\xfc\x01\u00c1\xfd\x01\u00c1\xfe\x01\u00c1\xff\x01\xe0\x94L*\xe4\x82Y5\x05\xf0\x16<\xde\xfc\a>\x81\xc6<\xdaA\a\x8a\x15-\x02\xc7\xe1J\xf6\x80\x00\x00\xe0\x94\xa8\xe8\xf1G2e\x8eKQ\xe8q\x191\x05:\x8ai\xba\xf2\xb1\x8a\x15-\x02\xc7\xe1J\xf6\x80\x00\x00\xe1\x94\u0665\x17\x9f\t\x1d\x85\x05\x1d<\x98'\x85\xef\xd1E\\\uc199\x8b\bE\x95\x16\x14\x01HJ\x00\x00\x00\xe1\x94\u08bdBX\xd2v\x887\xba\xa2j(\xfeq\xdc\a\x9f\x84\u01cbJG\xe3\xc1$H\xf4\xad\x00\x00\x00" -const calaverasAllocData = "\xf9\x06\x14\u0080\x01\xc2\x01\x01\xc2\x02\x01\xc2\x03\x01\xc2\x04\x01\xc2\x05\x01\xc2\x06\x01\xc2\a\x01\xc2\b\x01\xc2\t\x01\xc2\n\x01\xc2\v\x01\xc2\f\x01\xc2\r\x01\xc2\x0e\x01\xc2\x0f\x01\xc2\x10\x01\xc2\x11\x01\xc2\x12\x01\xc2\x13\x01\xc2\x14\x01\xc2\x15\x01\xc2\x16\x01\xc2\x17\x01\xc2\x18\x01\xc2\x19\x01\xc2\x1a\x01\xc2\x1b\x01\xc2\x1c\x01\xc2\x1d\x01\xc2\x1e\x01\xc2\x1f\x01\xc2 \x01\xc2!\x01\xc2\"\x01\xc2#\x01\xc2$\x01\xc2%\x01\xc2&\x01\xc2'\x01\xc2(\x01\xc2)\x01\xc2*\x01\xc2+\x01\xc2,\x01\xc2-\x01\xc2.\x01\xc2/\x01\xc20\x01\xc21\x01\xc22\x01\xc23\x01\xc24\x01\xc25\x01\xc26\x01\xc27\x01\xc28\x01\xc29\x01\xc2:\x01\xc2;\x01\xc2<\x01\xc2=\x01\xc2>\x01\xc2?\x01\xc2@\x01\xc2A\x01\xc2B\x01\xc2C\x01\xc2D\x01\xc2E\x01\xc2F\x01\xc2G\x01\xc2H\x01\xc2I\x01\xc2J\x01\xc2K\x01\xc2L\x01\xc2M\x01\xc2N\x01\xc2O\x01\xc2P\x01\xc2Q\x01\xc2R\x01\xc2S\x01\xc2T\x01\xc2U\x01\xc2V\x01\xc2W\x01\xc2X\x01\xc2Y\x01\xc2Z\x01\xc2[\x01\xc2\\\x01\xc2]\x01\xc2^\x01\xc2_\x01\xc2`\x01\xc2a\x01\xc2b\x01\xc2c\x01\xc2d\x01\xc2e\x01\xc2f\x01\xc2g\x01\xc2h\x01\xc2i\x01\xc2j\x01\xc2k\x01\xc2l\x01\xc2m\x01\xc2n\x01\xc2o\x01\xc2p\x01\xc2q\x01\xc2r\x01\xc2s\x01\xc2t\x01\xc2u\x01\xc2v\x01\xc2w\x01\xc2x\x01\xc2y\x01\xc2z\x01\xc2{\x01\xc2|\x01\xc2}\x01\xc2~\x01\xc2\u007f\x01\u00c1\x80\x01\u00c1\x81\x01\u00c1\x82\x01\u00c1\x83\x01\u00c1\x84\x01\u00c1\x85\x01\u00c1\x86\x01\u00c1\x87\x01\u00c1\x88\x01\u00c1\x89\x01\u00c1\x8a\x01\u00c1\x8b\x01\u00c1\x8c\x01\u00c1\x8d\x01\u00c1\x8e\x01\u00c1\x8f\x01\u00c1\x90\x01\u00c1\x91\x01\u00c1\x92\x01\u00c1\x93\x01\u00c1\x94\x01\u00c1\x95\x01\u00c1\x96\x01\u00c1\x97\x01\u00c1\x98\x01\u00c1\x99\x01\u00c1\x9a\x01\u00c1\x9b\x01\u00c1\x9c\x01\u00c1\x9d\x01\u00c1\x9e\x01\u00c1\x9f\x01\u00c1\xa0\x01\u00c1\xa1\x01\u00c1\xa2\x01\u00c1\xa3\x01\u00c1\xa4\x01\u00c1\xa5\x01\u00c1\xa6\x01\u00c1\xa7\x01\u00c1\xa8\x01\u00c1\xa9\x01\u00c1\xaa\x01\u00c1\xab\x01\u00c1\xac\x01\u00c1\xad\x01\u00c1\xae\x01\u00c1\xaf\x01\u00c1\xb0\x01\u00c1\xb1\x01\u00c1\xb2\x01\u00c1\xb3\x01\u00c1\xb4\x01\u00c1\xb5\x01\u00c1\xb6\x01\u00c1\xb7\x01\u00c1\xb8\x01\u00c1\xb9\x01\u00c1\xba\x01\u00c1\xbb\x01\u00c1\xbc\x01\u00c1\xbd\x01\u00c1\xbe\x01\u00c1\xbf\x01\u00c1\xc0\x01\u00c1\xc1\x01\u00c1\xc2\x01\u00c1\xc3\x01\u00c1\xc4\x01\u00c1\xc5\x01\u00c1\xc6\x01\u00c1\xc7\x01\u00c1\xc8\x01\u00c1\xc9\x01\u00c1\xca\x01\u00c1\xcb\x01\u00c1\xcc\x01\u00c1\xcd\x01\u00c1\xce\x01\u00c1\xcf\x01\u00c1\xd0\x01\u00c1\xd1\x01\u00c1\xd2\x01\u00c1\xd3\x01\u00c1\xd4\x01\u00c1\xd5\x01\u00c1\xd6\x01\u00c1\xd7\x01\u00c1\xd8\x01\u00c1\xd9\x01\u00c1\xda\x01\u00c1\xdb\x01\u00c1\xdc\x01\u00c1\xdd\x01\u00c1\xde\x01\u00c1\xdf\x01\u00c1\xe0\x01\u00c1\xe1\x01\u00c1\xe2\x01\u00c1\xe3\x01\u00c1\xe4\x01\u00c1\xe5\x01\u00c1\xe6\x01\u00c1\xe7\x01\u00c1\xe8\x01\u00c1\xe9\x01\u00c1\xea\x01\u00c1\xeb\x01\u00c1\xec\x01\u00c1\xed\x01\u00c1\xee\x01\u00c1\xef\x01\u00c1\xf0\x01\u00c1\xf1\x01\u00c1\xf2\x01\u00c1\xf3\x01\u00c1\xf4\x01\u00c1\xf5\x01\u00c1\xf6\x01\u00c1\xf7\x01\u00c1\xf8\x01\u00c1\xf9\x01\u00c1\xfa\x01\u00c1\xfb\x01\u00c1\xfc\x01\u00c1\xfd\x01\u00c1\xfe\x01\u00c1\xff\x01\xf6\x94\x0e\x89\xe2\xae\xdb\x1c\xfc\u06d4$\xd4\x1a\x1f!\x8fA2s\x81r\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94\x10A\xaf\xbc\xb3Y\u0568\xdcX\xc1[/\xf5\x13T\xff\x8a!}\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94#o\xf1\xe9t\x19\xae\x93\xad\x80\xca\xfb\xaa!\"\f]x\xfb}\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94`\xad\xc0\xf8\x9aA\xaf#|\xe75T\xed\xe1p\xd73\xec\x14\xe0\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94y\x9d2\x9e_X4\x19\x16|\xd7\"\x96$\x85\x92n3\x8fJ\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94|\xf5\xb7\x9b\xfe)\x1ag\xab\x02\xb3\x93\xe4V\xcc\xc4\xc2f\xf7S\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94\x8a\x8e\xaf\xb1\xcfb\xbf\xbe\xb1t\x17i\xda\xe1\xa9\xddG\x99a\x92\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94\x8b\xa1\xf1\tU\x1b\xd42\x800\x12dZ\xc16\xdd\xd6M\xbar\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94\xb0*.\xda\x1b1\u007f\xbd\x16v\x01(\x83k\n\u015bV\x0e\x9d\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94\xba\xdc\r\xe9\xe0yK\x04\x9b^\xa6<>\x1ei\x8a4v\xc1r\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94\xf00\v\ue24a\xe2r\xeb4~\x83i\xac\fv\xdfB\xc9?\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94\xfe;U~\x8f\xb6+\x89\xf4\x91kr\x1b\xe5\\\ub08d\xbds\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" const sepoliaAllocData = "\xf9\x01\xee\u0791i\x16\xa8{\x823?BE\x04f#\xb27\x94\xc6\\\x8b\bE\x95\x16\x14\x01HJ\x00\x00\x00\xe1\x94\x10\xf5\xd4XT\xe08\a\x14\x85\xac\x9e@#\b\u03c0\xd2\xd2\xfe\x8bR\xb7\xd2\xdc\xc8\f\xd2\xe4\x00\x00\x00\u0794y\x9d2\x9e_X4\x19\x16|\xd7\"\x96$\x85\x92n3\x8fJ\x88\r\u0db3\xa7d\x00\x00\xe0\x94|\xf5\xb7\x9b\xfe)\x1ag\xab\x02\xb3\x93\xe4V\xcc\xc4\xc2f\xf7S\x8a\xd3\xc2\x1b\xce\xcc\xed\xa1\x00\x00\x00\xe0\x94\x8b\u007f\tw\xbbO\x0f\xbepv\xfa\"\xbc$\xac\xa0CX?^\x8a\xd3\xc2\x1b\xce\xcc\xed\xa1\x00\x00\x00\xe0\x94\xa2\xa6\xd949\x14O\xfeM'\xc9\xe0\x88\xdc\u0637\x83\x94bc\x8a\xd3\xc2\x1b\xce\xcc\xed\xa1\x00\x00\x00\xe0\x94\xaa\xec\x869DA\xf9\x15\xbc\xe3\xe6\xab9\x99w\xe9\x90o;i\x8a\xd3\xc2\x1b\xce\xcc\xed\xa1\x00\x00\x00\u1532\x1c3\xde\x1f\xab?\xa1T\x99\xc6+Y\xfe\f\xc3%\x00 \u044bR\xb7\xd2\xdc\xc8\f\xd2\xe4\x00\x00\x00\xe0\x94\xbc\x11)Y6\xaay\u0554\x13\x9d\xe1\xb2\xe1&)AO;\u06ca\xd3\xc2\x1b\xce\xcc\xed\xa1\x00\x00\x00\xe0\x94\xbe\xef2\xca[\x9a\x19\x8d'\xb4\xe0/LpC\x9f\xe6\x03V\u03ca\xd3\xc2\x1b\xce\xcc\xed\xa1\x00\x00\x00\xe1\x94\xd7\xd7lX\xb3\xa5\x19\xe9\xfal\xc4\xd2-\xc0\x17%\x9b\u011f\x1e\x8bR\xb7\xd2\xdc\xc8\f\xd2\xe4\x00\x00\x00\xe0\x94\xd7\xed\xdbx\xed)[<\x96)$\x0e\x89$\xfb\x8d\x88t\xdd\u060a\xd3\xc2\x1b\xce\xcc\xed\xa1\x00\x00\x00\xe0\x94\u0665\x17\x9f\t\x1d\x85\x05\x1d<\x98'\x85\xef\xd1E\\\uc199\x8a\xd3\xc2\x1b\xce\xcc\xed\xa1\x00\x00\x00\xe0\x94\xe2\xe2e\x90(\x147\x84\xd5W\xbc\xeco\xf3\xa0r\x10H\x88\n\x8a\xd3\xc2\x1b\xce\xcc\xed\xa1\x00\x00\x00\xe0\x94\xf4|\xae\x1c\xf7\x9c\xa6u\x8b\xfcx}\xbd!\u6f7eq\x12\xb8\x8a\xd3\xc2\x1b\xce\xcc\xed\xa1\x00\x00\x00" const KilnAllocData = `{ "config": { diff --git a/core/genesis_test.go b/core/genesis_test.go index e8010e3d4ebd4..ba3423e32a08e 100644 --- a/core/genesis_test.go +++ b/core/genesis_test.go @@ -178,7 +178,7 @@ func TestGenesisHashes(t *testing.T) { t.Errorf("case: %d a), want: %s, got: %s", i, c.want.Hex(), have.Hex()) } // Test via ToBlock - if have := c.genesis.ToBlock(nil).Hash(); have != c.want { + if have := c.genesis.ToBlock().Hash(); have != c.want { t.Errorf("case: %d a), want: %s, got: %s", i, c.want.Hex(), have.Hex()) } } @@ -192,11 +192,7 @@ func TestGenesis_Commit(t *testing.T) { } db := rawdb.NewMemoryDatabase() - genesisBlock, err := genesis.Commit(db) - if err != nil { - t.Fatal(err) - } - + genesisBlock := genesis.MustCommit(db) if genesis.Difficulty != nil { t.Fatalf("assumption wrong") } @@ -221,12 +217,12 @@ func TestReadWriteGenesisAlloc(t *testing.T) { {1}: {Balance: big.NewInt(1), Storage: map[common.Hash]common.Hash{{1}: {1}}}, {2}: {Balance: big.NewInt(2), Storage: map[common.Hash]common.Hash{{2}: {2}}}, } - hash = common.HexToHash("0xdeadbeef") + hash, _ = alloc.deriveHash() ) - alloc.write(db, hash) + alloc.flush(db) var reload GenesisAlloc - err := reload.UnmarshalJSON(rawdb.ReadGenesisState(db, hash)) + err := reload.UnmarshalJSON(rawdb.ReadGenesisStateSpec(db, hash)) if err != nil { t.Fatalf("Failed to load genesis state %v", err) } diff --git a/core/rawdb/accessors_chain.go b/core/rawdb/accessors_chain.go index 8ea2e2ca72730..aeba3690d2286 100644 --- a/core/rawdb/accessors_chain.go +++ b/core/rawdb/accessors_chain.go @@ -37,7 +37,7 @@ import ( func ReadCanonicalHash(db ethdb.Reader, number uint64) common.Hash { var data []byte db.ReadAncients(func(reader ethdb.AncientReaderOp) error { - data, _ = reader.Ancient(freezerHashTable, number) + data, _ = reader.Ancient(chainFreezerHashTable, number) if len(data) == 0 { // Get it by hash from leveldb data, _ = db.Get(headerHashKey(number)) @@ -335,7 +335,7 @@ func ReadHeaderRange(db ethdb.Reader, number uint64, count uint64) []rlp.RawValu } // read remaining from ancients max := count * 700 - data, err := db.AncientRange(freezerHeaderTable, i+1-count, count, max) + data, err := db.AncientRange(chainFreezerHeaderTable, i+1-count, count, max) if err == nil && uint64(len(data)) == count { // the data is on the order [h, h+1, .., n] -- reordering needed for i := range data { @@ -352,7 +352,7 @@ func ReadHeaderRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValu // First try to look up the data in ancient database. Extra hash // comparison is necessary since ancient database only maintains // the canonical data. - data, _ = reader.Ancient(freezerHeaderTable, number) + data, _ = reader.Ancient(chainFreezerHeaderTable, number) if len(data) > 0 && crypto.Keccak256Hash(data) == hash { return nil } @@ -428,7 +428,7 @@ func deleteHeaderWithoutNumber(db ethdb.KeyValueWriter, hash common.Hash, number // isCanon is an internal utility method, to check whether the given number/hash // is part of the ancient (canon) set. func isCanon(reader ethdb.AncientReaderOp, number uint64, hash common.Hash) bool { - h, err := reader.Ancient(freezerHashTable, number) + h, err := reader.Ancient(chainFreezerHashTable, number) if err != nil { return false } @@ -444,7 +444,7 @@ func ReadBodyRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValue db.ReadAncients(func(reader ethdb.AncientReaderOp) error { // Check if the data is in ancients if isCanon(reader, number, hash) { - data, _ = reader.Ancient(freezerBodiesTable, number) + data, _ = reader.Ancient(chainFreezerBodiesTable, number) return nil } // If not, try reading from leveldb @@ -459,7 +459,7 @@ func ReadBodyRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValue func ReadCanonicalBodyRLP(db ethdb.Reader, number uint64) rlp.RawValue { var data []byte db.ReadAncients(func(reader ethdb.AncientReaderOp) error { - data, _ = reader.Ancient(freezerBodiesTable, number) + data, _ = reader.Ancient(chainFreezerBodiesTable, number) if len(data) > 0 { return nil } @@ -527,7 +527,7 @@ func ReadTdRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValue { db.ReadAncients(func(reader ethdb.AncientReaderOp) error { // Check if the data is in ancients if isCanon(reader, number, hash) { - data, _ = reader.Ancient(freezerDifficultyTable, number) + data, _ = reader.Ancient(chainFreezerDifficultyTable, number) return nil } // If not, try reading from leveldb @@ -587,7 +587,7 @@ func ReadReceiptsRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawVa db.ReadAncients(func(reader ethdb.AncientReaderOp) error { // Check if the data is in ancients if isCanon(reader, number, hash) { - data, _ = reader.Ancient(freezerReceiptTable, number) + data, _ = reader.Ancient(chainFreezerReceiptTable, number) return nil } // If not, try reading from leveldb @@ -819,19 +819,19 @@ func WriteAncientBlocks(db ethdb.AncientWriter, blocks []*types.Block, receipts func writeAncientBlock(op ethdb.AncientWriteOp, block *types.Block, header *types.Header, receipts []*types.ReceiptForStorage, td *big.Int) error { num := block.NumberU64() - if err := op.AppendRaw(freezerHashTable, num, block.Hash().Bytes()); err != nil { + if err := op.AppendRaw(chainFreezerHashTable, num, block.Hash().Bytes()); err != nil { return fmt.Errorf("can't add block %d hash: %v", num, err) } - if err := op.Append(freezerHeaderTable, num, header); err != nil { + if err := op.Append(chainFreezerHeaderTable, num, header); err != nil { return fmt.Errorf("can't append block header %d: %v", num, err) } - if err := op.Append(freezerBodiesTable, num, block.Body()); err != nil { + if err := op.Append(chainFreezerBodiesTable, num, block.Body()); err != nil { return fmt.Errorf("can't append block body %d: %v", num, err) } - if err := op.Append(freezerReceiptTable, num, receipts); err != nil { + if err := op.Append(chainFreezerReceiptTable, num, receipts); err != nil { return fmt.Errorf("can't append block %d receipts: %v", num, err) } - if err := op.Append(freezerDifficultyTable, num, td); err != nil { + if err := op.Append(chainFreezerDifficultyTable, num, td); err != nil { return fmt.Errorf("can't append block %d total difficulty: %v", num, err) } return nil diff --git a/core/rawdb/accessors_chain_test.go b/core/rawdb/accessors_chain_test.go index dbb13caa416c5..21d23e1f0c8b8 100644 --- a/core/rawdb/accessors_chain_test.go +++ b/core/rawdb/accessors_chain_test.go @@ -285,7 +285,7 @@ func TestTdStorage(t *testing.T) { func TestCanonicalMappingStorage(t *testing.T) { db := NewMemoryDatabase() - // Create a test canonical number and assinged hash to move around + // Create a test canonical number and assigned hash to move around hash, number := common.Hash{0: 0xff}, uint64(314) if entry := ReadCanonicalHash(db, number); entry != (common.Hash{}) { t.Fatalf("Non existent canonical mapping returned: %v", entry) diff --git a/core/rawdb/accessors_metadata.go b/core/rawdb/accessors_metadata.go index f5a161adb6889..7a9e6442f0115 100644 --- a/core/rawdb/accessors_metadata.go +++ b/core/rawdb/accessors_metadata.go @@ -81,15 +81,16 @@ func WriteChainConfig(db ethdb.KeyValueWriter, hash common.Hash, cfg *params.Cha } } -// ReadGenesisState retrieves the genesis state based on the given genesis hash. -func ReadGenesisState(db ethdb.KeyValueReader, hash common.Hash) []byte { - data, _ := db.Get(genesisKey(hash)) +// ReadGenesisStateSpec retrieves the genesis state specification based on the +// given genesis hash. +func ReadGenesisStateSpec(db ethdb.KeyValueReader, hash common.Hash) []byte { + data, _ := db.Get(genesisStateSpecKey(hash)) return data } -// WriteGenesisState writes the genesis state into the disk. -func WriteGenesisState(db ethdb.KeyValueWriter, hash common.Hash, data []byte) { - if err := db.Put(genesisKey(hash), data); err != nil { +// WriteGenesisStateSpec writes the genesis state specification into the disk. +func WriteGenesisStateSpec(db ethdb.KeyValueWriter, hash common.Hash, data []byte) { + if err := db.Put(genesisStateSpecKey(hash), data); err != nil { log.Crit("Failed to store genesis state", "err", err) } } diff --git a/core/rawdb/ancient_scheme.go b/core/rawdb/ancient_scheme.go new file mode 100644 index 0000000000000..3da061cbd977c --- /dev/null +++ b/core/rawdb/ancient_scheme.go @@ -0,0 +1,86 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rawdb + +import "fmt" + +// The list of table names of chain freezer. +const ( + // chainFreezerHeaderTable indicates the name of the freezer header table. + chainFreezerHeaderTable = "headers" + + // chainFreezerHashTable indicates the name of the freezer canonical hash table. + chainFreezerHashTable = "hashes" + + // chainFreezerBodiesTable indicates the name of the freezer block body table. + chainFreezerBodiesTable = "bodies" + + // chainFreezerReceiptTable indicates the name of the freezer receipts table. + chainFreezerReceiptTable = "receipts" + + // chainFreezerDifficultyTable indicates the name of the freezer total difficulty table. + chainFreezerDifficultyTable = "diffs" +) + +// chainFreezerNoSnappy configures whether compression is disabled for the ancient-tables. +// Hashes and difficulties don't compress well. +var chainFreezerNoSnappy = map[string]bool{ + chainFreezerHeaderTable: false, + chainFreezerHashTable: true, + chainFreezerBodiesTable: false, + chainFreezerReceiptTable: false, + chainFreezerDifficultyTable: true, +} + +// The list of identifiers of ancient stores. +var ( + chainFreezerName = "chain" // the folder name of chain segment ancient store. +) + +// freezers the collections of all builtin freezers. +var freezers = []string{chainFreezerName} + +// InspectFreezerTable dumps out the index of a specific freezer table. The passed +// ancient indicates the path of root ancient directory where the chain freezer can +// be opened. Start and end specify the range for dumping out indexes. +// Note this function can only be used for debugging purposes. +func InspectFreezerTable(ancient string, freezerName string, tableName string, start, end int64) error { + var ( + path string + tables map[string]bool + ) + switch freezerName { + case chainFreezerName: + path, tables = resolveChainFreezerDir(ancient), chainFreezerNoSnappy + default: + return fmt.Errorf("unknown freezer, supported ones: %v", freezers) + } + noSnappy, exist := tables[tableName] + if !exist { + var names []string + for name := range tables { + names = append(names, name) + } + return fmt.Errorf("unknown table, supported ones: %v", names) + } + table, err := newFreezerTable(path, tableName, noSnappy, true) + if err != nil { + return err + } + table.dumpIndexStdout(start, end) + return nil +} diff --git a/core/rawdb/chain_freezer.go b/core/rawdb/chain_freezer.go index 4c49db2748b22..7d9c9c0156490 100644 --- a/core/rawdb/chain_freezer.go +++ b/core/rawdb/chain_freezer.go @@ -241,7 +241,7 @@ func (f *chainFreezer) freeze(db ethdb.KeyValueStore) { if n := len(ancients); n > 0 { context = append(context, []interface{}{"hash", ancients[n-1]}...) } - log.Info("Deep froze chain segment", context...) + log.Debug("Deep froze chain segment", context...) // Avoid database thrashing with tiny writes if frozen-first < freezerBatchLimit { @@ -278,19 +278,19 @@ func (f *chainFreezer) freezeRange(nfdb *nofreezedb, number, limit uint64) (hash } // Write to the batch. - if err := op.AppendRaw(freezerHashTable, number, hash[:]); err != nil { + if err := op.AppendRaw(chainFreezerHashTable, number, hash[:]); err != nil { return fmt.Errorf("can't write hash to Freezer: %v", err) } - if err := op.AppendRaw(freezerHeaderTable, number, header); err != nil { + if err := op.AppendRaw(chainFreezerHeaderTable, number, header); err != nil { return fmt.Errorf("can't write header to Freezer: %v", err) } - if err := op.AppendRaw(freezerBodiesTable, number, body); err != nil { + if err := op.AppendRaw(chainFreezerBodiesTable, number, body); err != nil { return fmt.Errorf("can't write body to Freezer: %v", err) } - if err := op.AppendRaw(freezerReceiptTable, number, receipts); err != nil { + if err := op.AppendRaw(chainFreezerReceiptTable, number, receipts); err != nil { return fmt.Errorf("can't write receipts to Freezer: %v", err) } - if err := op.AppendRaw(freezerDifficultyTable, number, td); err != nil { + if err := op.AppendRaw(chainFreezerDifficultyTable, number, td); err != nil { return fmt.Errorf("can't write td to Freezer: %v", err) } diff --git a/core/rawdb/chain_iterator.go b/core/rawdb/chain_iterator.go index 21e42f42d43ac..867fed63ad920 100644 --- a/core/rawdb/chain_iterator.go +++ b/core/rawdb/chain_iterator.go @@ -50,7 +50,7 @@ func InitDatabaseFromFreezer(db ethdb.Database) { if i+count > frozen { count = frozen - i } - data, err := db.AncientRange(freezerHashTable, i, count, 32*count) + data, err := db.AncientRange(chainFreezerHashTable, i, count, 32*count) if err != nil { log.Crit("Failed to init database from freezer", "err", err) } diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 2b870d16d44f9..1eaf033bbefa5 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -21,6 +21,7 @@ import ( "errors" "fmt" "os" + "path" "sync/atomic" "time" @@ -34,10 +35,16 @@ import ( // freezerdb is a database wrapper that enabled freezer data retrievals. type freezerdb struct { + ancientRoot string ethdb.KeyValueStore ethdb.AncientStore } +// AncientDatadir returns the path of root ancient directory. +func (frdb *freezerdb) AncientDatadir() (string, error) { + return frdb.ancientRoot, nil +} + // Close implements io.Closer, closing both the fast key-value store as well as // the slow ancient tables. func (frdb *freezerdb) Close() error { @@ -162,12 +169,36 @@ func NewDatabase(db ethdb.KeyValueStore) ethdb.Database { return &nofreezedb{KeyValueStore: db} } +// resolveChainFreezerDir is a helper function which resolves the absolute path +// of chain freezer by considering backward compatibility. +func resolveChainFreezerDir(ancient string) string { + // Check if the chain freezer is already present in the specified + // sub folder, if not then two possibilities: + // - chain freezer is not initialized + // - chain freezer exists in legacy location (root ancient folder) + freezer := path.Join(ancient, chainFreezerName) + if !common.FileExist(freezer) { + if !common.FileExist(ancient) { + // The entire ancient store is not initialized, still use the sub + // folder for initialization. + } else { + // Ancient root is already initialized, then we hold the assumption + // that chain freezer is also initialized and located in root folder. + // In this case fallback to legacy location. + freezer = ancient + log.Info("Found legacy ancient chain path", "location", ancient) + } + } + return freezer +} + // NewDatabaseWithFreezer creates a high level database on top of a given key- // value data store with a freezer moving immutable chain segments into cold -// storage. -func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace string, readonly bool) (ethdb.Database, error) { +// storage. The passed ancient indicates the path of root ancient directory +// where the chain freezer can be opened. +func NewDatabaseWithFreezer(db ethdb.KeyValueStore, ancient string, namespace string, readonly bool) (ethdb.Database, error) { // Create the idle freezer instance - frdb, err := newChainFreezer(freezer, namespace, readonly, freezerTableSize, FreezerNoSnappy) + frdb, err := newChainFreezer(resolveChainFreezerDir(ancient), namespace, readonly, freezerTableSize, chainFreezerNoSnappy) if err != nil { return nil, err } @@ -198,7 +229,7 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace st // If the freezer already contains something, ensure that the genesis blocks // match, otherwise we might mix up freezers across chains and destroy both // the freezer and the key-value store. - frgenesis, err := frdb.Ancient(freezerHashTable, 0) + frgenesis, err := frdb.Ancient(chainFreezerHashTable, 0) if err != nil { return nil, fmt.Errorf("failed to retrieve genesis from ancient %v", err) } else if !bytes.Equal(kvgenesis, frgenesis) { @@ -208,7 +239,7 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace st // are contiguous, otherwise we might end up with a non-functional freezer. if kvhash, _ := db.Get(headerHashKey(frozen)); len(kvhash) == 0 { // Subsequent header after the freezer limit is missing from the database. - // Reject startup is the database has a more recent head. + // Reject startup if the database has a more recent head. if *ReadHeaderNumber(db, ReadHeadHeaderHash(db)) > frozen-1 { return nil, fmt.Errorf("gap (#%d) in the chain between ancients and leveldb", frozen) } @@ -229,7 +260,7 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace st if kvblob, _ := db.Get(headerHashKey(1)); len(kvblob) == 0 { return nil, errors.New("ancient chain segments already extracted, please set --datadir.ancient to the correct path") } - // Block #1 is still in the database, we're allowed to init a new feezer + // Block #1 is still in the database, we're allowed to init a new freezer } // Otherwise, the head header is still the genesis, we're allowed to init a new // freezer. @@ -244,6 +275,7 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace st }() } return &freezerdb{ + ancientRoot: ancient, KeyValueStore: db, AncientStore: frdb, }, nil @@ -273,13 +305,15 @@ func NewLevelDBDatabase(file string, cache int, handles int, namespace string, r } // NewLevelDBDatabaseWithFreezer creates a persistent key-value database with a -// freezer moving immutable chain segments into cold storage. -func NewLevelDBDatabaseWithFreezer(file string, cache int, handles int, freezer string, namespace string, readonly bool) (ethdb.Database, error) { +// freezer moving immutable chain segments into cold storage. The passed ancient +// indicates the path of root ancient directory where the chain freezer can be +// opened. +func NewLevelDBDatabaseWithFreezer(file string, cache int, handles int, ancient string, namespace string, readonly bool) (ethdb.Database, error) { kvdb, err := leveldb.New(file, cache, handles, namespace, readonly) if err != nil { return nil, err } - frdb, err := NewDatabaseWithFreezer(kvdb, freezer, namespace, readonly) + frdb, err := NewDatabaseWithFreezer(kvdb, ancient, namespace, readonly) if err != nil { kvdb.Close() return nil, err @@ -441,7 +475,7 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { } // Inspect append-only file store then. ancientSizes := []*common.StorageSize{&ancientHeadersSize, &ancientBodiesSize, &ancientReceiptsSize, &ancientHashesSize, &ancientTdsSize} - for i, category := range []string{freezerHeaderTable, freezerBodiesTable, freezerReceiptTable, freezerHashTable, freezerDifficultyTable} { + for i, category := range []string{chainFreezerHeaderTable, chainFreezerBodiesTable, chainFreezerReceiptTable, chainFreezerHashTable, chainFreezerDifficultyTable} { if size, err := db.AncientSize(category); err == nil { *ancientSizes[i] += common.StorageSize(size) total += common.StorageSize(size) diff --git a/core/rawdb/freezer.go b/core/rawdb/freezer.go index 63fd8cdcf86f5..6dea98c3d3c4b 100644 --- a/core/rawdb/freezer.go +++ b/core/rawdb/freezer.go @@ -68,8 +68,6 @@ type Freezer struct { frozen uint64 // Number of blocks already frozen tail uint64 // Number of the first stored item in the freezer - datadir string // Path of root directory of ancient store - // This lock synchronizes writers and the truncate operation, as well as // the "atomic" (batched) read operations. writeLock sync.RWMutex @@ -111,7 +109,6 @@ func NewFreezer(datadir string, namespace string, readonly bool, maxTableSize ui readonly: readonly, tables: make(map[string]*freezerTable), instanceLock: lock, - datadir: datadir, } // Create the tables. @@ -429,7 +426,7 @@ func (f *Freezer) MigrateTable(kind string, convert convertLegacyFn) error { // Set up new dir for the migrated table, the content of which // we'll at the end move over to the ancients dir. migrationPath := filepath.Join(ancientsPath, "migration") - newTable, err := NewFreezerTable(migrationPath, kind, table.noCompression, false) + newTable, err := newFreezerTable(migrationPath, kind, table.noCompression, false) if err != nil { return err } @@ -486,11 +483,5 @@ func (f *Freezer) MigrateTable(kind string, convert convertLegacyFn) error { if err := os.Remove(migrationPath); err != nil { return err } - return nil } - -// AncientDatadir returns the root directory path of the ancient store. -func (f *Freezer) AncientDatadir() (string, error) { - return f.datadir, nil -} diff --git a/core/rawdb/freezer_table.go b/core/rawdb/freezer_table.go index dd4a80efcbc57..3fe691cf6d2a7 100644 --- a/core/rawdb/freezer_table.go +++ b/core/rawdb/freezer_table.go @@ -46,7 +46,7 @@ var ( errNotSupported = errors.New("this operation is not supported") ) -// indexEntry contains the number/id of the file that the data resides in, aswell as the +// indexEntry contains the number/id of the file that the data resides in, as well as the // offset within the file to the end of the data. // In serialized form, the filenum is stored as uint16. type indexEntry struct { @@ -123,8 +123,8 @@ type freezerTable struct { lock sync.RWMutex // Mutex protecting the data file descriptors } -// NewFreezerTable opens the given path as a freezer table. -func NewFreezerTable(path, name string, disableSnappy, readonly bool) (*freezerTable, error) { +// newFreezerTable opens the given path as a freezer table. +func newFreezerTable(path, name string, disableSnappy, readonly bool) (*freezerTable, error) { return newTable(path, name, metrics.NilMeter{}, metrics.NilMeter{}, metrics.NilGauge{}, freezerTableSize, disableSnappy, readonly) } @@ -884,9 +884,7 @@ func (t *freezerTable) Sync() error { return t.head.Sync() } -// DumpIndex is a debug print utility function, mainly for testing. It can also -// be used to analyse a live freezer table index. -func (t *freezerTable) DumpIndex(start, stop int64) { +func (t *freezerTable) dumpIndexStdout(start, stop int64) { t.dumpIndex(os.Stdout, start, stop) } diff --git a/core/rawdb/freezer_table_test.go b/core/rawdb/freezer_table_test.go index 0bddcf7211363..ea28e71756de6 100644 --- a/core/rawdb/freezer_table_test.go +++ b/core/rawdb/freezer_table_test.go @@ -902,7 +902,7 @@ func TestSequentialRead(t *testing.T) { } // Write 15 bytes 30 times writeChunks(t, f, 30, 15) - f.DumpIndex(0, 30) + f.dumpIndexStdout(0, 30) f.Close() } { // Open it, iterate, verify iteration diff --git a/core/rawdb/freezer_test.go b/core/rawdb/freezer_test.go index c28d35ef387d0..630911ec867c7 100644 --- a/core/rawdb/freezer_test.go +++ b/core/rawdb/freezer_test.go @@ -277,7 +277,7 @@ func TestFreezerReadonlyValidate(t *testing.T) { // Re-openening as readonly should fail when validating // table lengths. - f, err = NewFreezer(dir, "", true, 2049, tables) + _, err = NewFreezer(dir, "", true, 2049, tables) if err == nil { t.Fatal("readonly freezer should fail with differing table lengths") } diff --git a/core/rawdb/freezer_utils_test.go b/core/rawdb/freezer_utils_test.go index cc300cb614fa0..829cbfb4f332d 100644 --- a/core/rawdb/freezer_utils_test.go +++ b/core/rawdb/freezer_utils_test.go @@ -43,7 +43,7 @@ func TestCopyFrom(t *testing.T) { {"foo", "bar", 8, true}, } for _, c := range cases { - os.WriteFile(c.src, content, 0644) + os.WriteFile(c.src, content, 0600) if err := copyFrom(c.src, c.dest, c.offset, func(f *os.File) error { if !c.writePrefix { diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index 041c9f0449671..d5f751da3a13c 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -111,33 +111,6 @@ var ( preimageHitCounter = metrics.NewRegisteredCounter("db/preimage/hits", nil) ) -const ( - // freezerHeaderTable indicates the name of the freezer header table. - freezerHeaderTable = "headers" - - // freezerHashTable indicates the name of the freezer canonical hash table. - freezerHashTable = "hashes" - - // freezerBodiesTable indicates the name of the freezer block body table. - freezerBodiesTable = "bodies" - - // freezerReceiptTable indicates the name of the freezer receipts table. - freezerReceiptTable = "receipts" - - // freezerDifficultyTable indicates the name of the freezer total difficulty table. - freezerDifficultyTable = "diffs" -) - -// FreezerNoSnappy configures whether compression is disabled for the ancient-tables. -// Hashes and difficulties don't compress well. -var FreezerNoSnappy = map[string]bool{ - freezerHeaderTable: false, - freezerHashTable: true, - freezerBodiesTable: false, - freezerReceiptTable: false, - freezerDifficultyTable: true, -} - // LegacyTxLookupEntry is the legacy TxLookupEntry definition with some unnecessary // fields. type LegacyTxLookupEntry struct { @@ -247,7 +220,7 @@ func configKey(hash common.Hash) []byte { return append(configPrefix, hash.Bytes()...) } -// genesisKey = genesisPrefix + hash -func genesisKey(hash common.Hash) []byte { +// genesisStateSpecKey = genesisPrefix + hash +func genesisStateSpecKey(hash common.Hash) []byte { return append(genesisPrefix, hash.Bytes()...) } diff --git a/core/state/database.go b/core/state/database.go index bbcd2358e5b8e..96b6bcfe6551a 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -63,7 +63,7 @@ type Trie interface { // GetKey returns the sha3 preimage of a hashed key that was previously used // to store a value. // - // TODO(fjl): remove this when SecureTrie is removed + // TODO(fjl): remove this when StateTrie is removed GetKey([]byte) []byte // TryGet returns the value for key stored in the trie. The value bytes must @@ -71,8 +71,8 @@ type Trie interface { // trie.MissingNodeError is returned. TryGet(key []byte) ([]byte, error) - // TryUpdateAccount abstract an account write in the trie. - TryUpdateAccount(key []byte, account *types.StateAccount) error + // TryGetAccount abstract an account read from the trie. + TryGetAccount(key []byte) (*types.StateAccount, error) // TryUpdate associates key with value in the trie. If value has length zero, any // existing value is deleted from the trie. The value bytes must not be modified @@ -80,17 +80,27 @@ type Trie interface { // database, a trie.MissingNodeError is returned. TryUpdate(key, value []byte) error + // TryUpdateAccount abstract an account write to the trie. + TryUpdateAccount(key []byte, account *types.StateAccount) error + // TryDelete removes any existing value for key from the trie. If a node was not // found in the database, a trie.MissingNodeError is returned. TryDelete(key []byte) error + // TryDeleteAccount abstracts an account deletion from the trie. + TryDeleteAccount(key []byte) error + // Hash returns the root hash of the trie. It does not write to the database and // can be used even if the trie doesn't have one. Hash() common.Hash - // Commit writes all nodes to the trie's memory database, tracking the internal - // and external (for account tries) references. - Commit(onleaf trie.LeafCallback) (common.Hash, int, error) + // Commit collects all dirty nodes in the trie and replace them with the + // corresponding node hash. All collected nodes(including dirty leaves if + // collectLeaf is true) will be encapsulated into a nodeset for return. + // The returned nodeset can be nil if the trie is clean(nothing to commit). + // Once the trie is committed, it's not usable anymore. A new trie must + // be created with new root and updated trie database for following usage + Commit(collectLeaf bool) (common.Hash, *trie.NodeSet, error) // NodeIterator returns an iterator that returns nodes of the trie. Iteration // starts at the key after the given start key. @@ -133,7 +143,7 @@ type cachingDB struct { // OpenTrie opens the main account trie at a specific root hash. func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) { - tr, err := trie.NewSecure(root, db.db) + tr, err := trie.NewStateTrie(common.Hash{}, root, db.db) if err != nil { return nil, err } @@ -142,7 +152,7 @@ func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) { // OpenStorageTrie opens the storage trie of an account. func (db *cachingDB) OpenStorageTrie(addrHash, root common.Hash) (Trie, error) { - tr, err := trie.NewSecure(root, db.db) + tr, err := trie.NewStateTrie(addrHash, root, db.db) if err != nil { return nil, err } @@ -152,7 +162,7 @@ func (db *cachingDB) OpenStorageTrie(addrHash, root common.Hash) (Trie, error) { // CopyTrie returns an independent copy of the given trie. func (db *cachingDB) CopyTrie(t Trie) Trie { switch t := t.(type) { - case *trie.SecureTrie: + case *trie.StateTrie: return t.Copy() default: panic(fmt.Errorf("unknown trie type %T", t)) diff --git a/core/state/metrics.go b/core/state/metrics.go index 7b40ff37aff0e..35d2df92dda4b 100644 --- a/core/state/metrics.go +++ b/core/state/metrics.go @@ -19,10 +19,10 @@ package state import "github.com/ethereum/go-ethereum/metrics" var ( - accountUpdatedMeter = metrics.NewRegisteredMeter("state/update/account", nil) - storageUpdatedMeter = metrics.NewRegisteredMeter("state/update/storage", nil) - accountDeletedMeter = metrics.NewRegisteredMeter("state/delete/account", nil) - storageDeletedMeter = metrics.NewRegisteredMeter("state/delete/storage", nil) - accountCommittedMeter = metrics.NewRegisteredMeter("state/commit/account", nil) - storageCommittedMeter = metrics.NewRegisteredMeter("state/commit/storage", nil) + accountUpdatedMeter = metrics.NewRegisteredMeter("state/update/account", nil) + storageUpdatedMeter = metrics.NewRegisteredMeter("state/update/storage", nil) + accountDeletedMeter = metrics.NewRegisteredMeter("state/delete/account", nil) + storageDeletedMeter = metrics.NewRegisteredMeter("state/delete/storage", nil) + accountTrieCommittedMeter = metrics.NewRegisteredMeter("state/commit/accountnodes", nil) + storageTriesCommittedMeter = metrics.NewRegisteredMeter("state/commit/storagenodes", nil) ) diff --git a/core/state/pruner/bloom.go b/core/state/pruner/bloom.go index 29bc4e7314efa..72315db720f19 100644 --- a/core/state/pruner/bloom.go +++ b/core/state/pruner/bloom.go @@ -39,7 +39,7 @@ func (f stateBloomHasher) BlockSize() int { panic("not implem func (f stateBloomHasher) Size() int { return 8 } func (f stateBloomHasher) Sum64() uint64 { return binary.BigEndian.Uint64(f) } -// stateBloom is a bloom filter used during the state convesion(snapshot->state). +// stateBloom is a bloom filter used during the state conversion(snapshot->state). // The keys of all generated entries will be recorded here so that in the pruning // stage the entries belong to the specific version can be avoided for deletion. // @@ -100,7 +100,7 @@ func (bloom *stateBloom) Commit(filename, tempname string) error { } f.Close() - // Move the teporary file into it's final location + // Move the temporary file into it's final location return os.Rename(tempname, filename) } diff --git a/core/state/pruner/pruner.go b/core/state/pruner/pruner.go index a121839bd099f..2da2eda8b74df 100644 --- a/core/state/pruner/pruner.go +++ b/core/state/pruner/pruner.go @@ -410,7 +410,7 @@ func extractGenesis(db ethdb.Database, stateBloom *stateBloom) error { if genesis == nil { return errors.New("missing genesis block") } - t, err := trie.NewSecure(genesis.Root(), trie.NewDatabase(db)) + t, err := trie.NewStateTrie(common.Hash{}, genesis.Root(), trie.NewDatabase(db)) if err != nil { return err } @@ -430,7 +430,7 @@ func extractGenesis(db ethdb.Database, stateBloom *stateBloom) error { return err } if acc.Root != emptyRoot { - storageTrie, err := trie.NewSecure(acc.Root, trie.NewDatabase(db)) + storageTrie, err := trie.NewStateTrie(common.BytesToHash(accIter.LeafKey()), acc.Root, trie.NewDatabase(db)) if err != nil { return err } diff --git a/core/state/snapshot/conversion.go b/core/state/snapshot/conversion.go index f70cbf1e686b1..0f3934cb423bb 100644 --- a/core/state/snapshot/conversion.go +++ b/core/state/snapshot/conversion.go @@ -43,7 +43,7 @@ type trieKV struct { type ( // trieGeneratorFn is the interface of trie generation which can // be implemented by different trie algorithm. - trieGeneratorFn func(db ethdb.KeyValueWriter, in chan (trieKV), out chan (common.Hash)) + trieGeneratorFn func(db ethdb.KeyValueWriter, owner common.Hash, in chan (trieKV), out chan (common.Hash)) // leafCallbackFn is the callback invoked at the leaves of the trie, // returns the subtrie root with the specified subtrie identifier. @@ -253,7 +253,7 @@ func generateTrieRoot(db ethdb.KeyValueWriter, it Iterator, account common.Hash, wg.Add(1) go func() { defer wg.Done() - generatorFn(db, in, out) + generatorFn(db, account, in, out) }() // Spin up a go-routine for progress logging if report && stats != nil { @@ -360,8 +360,8 @@ func generateTrieRoot(db ethdb.KeyValueWriter, it Iterator, account common.Hash, return stop(nil) } -func stackTrieGenerate(db ethdb.KeyValueWriter, in chan trieKV, out chan common.Hash) { - t := trie.NewStackTrie(db) +func stackTrieGenerate(db ethdb.KeyValueWriter, owner common.Hash, in chan trieKV, out chan common.Hash) { + t := trie.NewStackTrieWithOwner(db, owner) for leaf := range in { t.TryUpdate(leaf.key[:], leaf.value) } diff --git a/core/state/snapshot/difflayer_test.go b/core/state/snapshot/difflayer_test.go index e15c1d5049b03..59db920481b01 100644 --- a/core/state/snapshot/difflayer_test.go +++ b/core/state/snapshot/difflayer_test.go @@ -332,7 +332,6 @@ func BenchmarkFlatten(b *testing.B) { value := make([]byte, 32) rand.Read(value) accStorage[randomHash()] = value - } storage[accountKey] = accStorage } @@ -382,7 +381,6 @@ func BenchmarkJournal(b *testing.B) { value := make([]byte, 32) rand.Read(value) accStorage[randomHash()] = value - } storage[accountKey] = accStorage } diff --git a/core/state/snapshot/disklayer_test.go b/core/state/snapshot/disklayer_test.go index b078951c72aa0..f95b79851598e 100644 --- a/core/state/snapshot/disklayer_test.go +++ b/core/state/snapshot/disklayer_test.go @@ -23,7 +23,6 @@ import ( "github.com/VictoriaMetrics/fastcache" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/ethdb/leveldb" "github.com/ethereum/go-ethereum/ethdb/memorydb" "github.com/ethereum/go-ethereum/rlp" ) @@ -515,11 +514,7 @@ func TestDiskMidAccountPartialMerge(t *testing.T) { // TestDiskSeek tests that seek-operations work on the disk layer func TestDiskSeek(t *testing.T) { // Create some accounts in the disk layer - diskdb, err := leveldb.New(t.TempDir(), 256, 0, "", false) - if err != nil { - t.Fatal(err) - } - db := rawdb.NewDatabase(diskdb) + db := rawdb.NewMemoryDatabase() defer db.Close() // Fill even keys [0,2,4...] diff --git a/core/state/snapshot/generate.go b/core/state/snapshot/generate.go index 769989aec21c0..bf714db4c2d01 100644 --- a/core/state/snapshot/generate.go +++ b/core/state/snapshot/generate.go @@ -166,7 +166,7 @@ func (result *proofResult) forEach(callback func(key []byte, val []byte) error) // // The proof result will be returned if the range proving is finished, otherwise // the error will be returned to abort the entire procedure. -func (dl *diskLayer) proveRange(ctx *generatorContext, root common.Hash, prefix []byte, kind string, origin []byte, max int, valueConvertFn func([]byte) ([]byte, error)) (*proofResult, error) { +func (dl *diskLayer) proveRange(ctx *generatorContext, owner common.Hash, root common.Hash, prefix []byte, kind string, origin []byte, max int, valueConvertFn func([]byte) ([]byte, error)) (*proofResult, error) { var ( keys [][]byte vals [][]byte @@ -234,7 +234,7 @@ func (dl *diskLayer) proveRange(ctx *generatorContext, root common.Hash, prefix // The snap state is exhausted, pass the entire key/val set for verification if origin == nil && !diskMore { - stackTr := trie.NewStackTrie(nil) + stackTr := trie.NewStackTrieWithOwner(nil, owner) for i, key := range keys { stackTr.TryUpdate(key, vals[i]) } @@ -248,7 +248,7 @@ func (dl *diskLayer) proveRange(ctx *generatorContext, root common.Hash, prefix return &proofResult{keys: keys, vals: vals}, nil } // Snap state is chunked, generate edge proofs for verification. - tr, err := trie.New(root, dl.triedb) + tr, err := trie.New(owner, root, dl.triedb) if err != nil { ctx.stats.Log("Trie missing, state snapshotting paused", dl.root, dl.genMarker) return nil, errMissingTrie @@ -313,9 +313,9 @@ type onStateCallback func(key []byte, val []byte, write bool, delete bool) error // generateRange generates the state segment with particular prefix. Generation can // either verify the correctness of existing state through range-proof and skip // generation, or iterate trie to regenerate state on demand. -func (dl *diskLayer) generateRange(ctx *generatorContext, root common.Hash, prefix []byte, kind string, origin []byte, max int, onState onStateCallback, valueConvertFn func([]byte) ([]byte, error)) (bool, []byte, error) { +func (dl *diskLayer) generateRange(ctx *generatorContext, owner common.Hash, root common.Hash, prefix []byte, kind string, origin []byte, max int, onState onStateCallback, valueConvertFn func([]byte) ([]byte, error)) (bool, []byte, error) { // Use range prover to check the validity of the flat state in the range - result, err := dl.proveRange(ctx, root, prefix, kind, origin, max, valueConvertFn) + result, err := dl.proveRange(ctx, owner, root, prefix, kind, origin, max, valueConvertFn) if err != nil { return false, nil, err } @@ -363,18 +363,21 @@ func (dl *diskLayer) generateRange(ctx *generatorContext, root common.Hash, pref if len(result.keys) > 0 { snapNodeCache = memorydb.New() snapTrieDb := trie.NewDatabase(snapNodeCache) - snapTrie, _ := trie.New(common.Hash{}, snapTrieDb) + snapTrie, _ := trie.New(owner, common.Hash{}, snapTrieDb) for i, key := range result.keys { snapTrie.Update(key, result.vals[i]) } - root, _, _ := snapTrie.Commit(nil) + root, nodes, _ := snapTrie.Commit(false) + if nodes != nil { + snapTrieDb.Update(trie.NewWithNodeSet(nodes)) + } snapTrieDb.Commit(root, false, nil) } // Construct the trie for state iteration, reuse the trie // if it's already opened with some nodes resolved. tr := result.tr if tr == nil { - tr, err = trie.New(root, dl.triedb) + tr, err = trie.New(owner, root, dl.triedb) if err != nil { ctx.stats.Log("Trie missing, state snapshotting paused", dl.root, dl.genMarker) return false, nil, errMissingTrie @@ -537,7 +540,7 @@ func generateStorages(ctx *generatorContext, dl *diskLayer, account common.Hash, // Loop for re-generating the missing storage slots. var origin = common.CopyBytes(storeMarker) for { - exhausted, last, err := dl.generateRange(ctx, storageRoot, append(rawdb.SnapshotStoragePrefix, account.Bytes()...), snapStorage, origin, storageCheckRange, onStorage, nil) + exhausted, last, err := dl.generateRange(ctx, account, storageRoot, append(rawdb.SnapshotStoragePrefix, account.Bytes()...), snapStorage, origin, storageCheckRange, onStorage, nil) if err != nil { return err // The procedure it aborted, either by external signal or internal error. } @@ -637,7 +640,7 @@ func generateAccounts(ctx *generatorContext, dl *diskLayer, accMarker []byte) er } origin := common.CopyBytes(accMarker) for { - exhausted, last, err := dl.generateRange(ctx, dl.root, rawdb.SnapshotAccountPrefix, snapAccount, origin, accountRange, onAccount, FullAccountRLP) + exhausted, last, err := dl.generateRange(ctx, common.Hash{}, dl.root, rawdb.SnapshotAccountPrefix, snapAccount, origin, accountRange, onAccount, FullAccountRLP) if err != nil { return err // The procedure it aborted, either by external signal or internal error. } diff --git a/core/state/snapshot/generate_test.go b/core/state/snapshot/generate_test.go index 94caed08ad7a7..5e5ded61ea2fa 100644 --- a/core/state/snapshot/generate_test.go +++ b/core/state/snapshot/generate_test.go @@ -26,47 +26,40 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/ethdb/memorydb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" "golang.org/x/crypto/sha3" ) +func hashData(input []byte) common.Hash { + var hasher = sha3.NewLegacyKeccak256() + var hash common.Hash + hasher.Reset() + hasher.Write(input) + hasher.Sum(hash[:0]) + return hash +} + // Tests that snapshot generation from an empty database. func TestGeneration(t *testing.T) { // We can't use statedb to make a test trie (circular dependency), so make // a fake one manually. We're going with a small account trie of 3 accounts, // two of which also has the same 3-slot storage trie attached. - var ( - diskdb = memorydb.New() - triedb = trie.NewDatabase(diskdb) - ) - stTrie, _ := trie.NewSecure(common.Hash{}, triedb) - stTrie.Update([]byte("key-1"), []byte("val-1")) // 0x1314700b81afc49f94db3623ef1df38f3ed18b73a1b7ea2f6c095118cf6118a0 - stTrie.Update([]byte("key-2"), []byte("val-2")) // 0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371 - stTrie.Update([]byte("key-3"), []byte("val-3")) // 0x51c71a47af0695957647fb68766d0becee77e953df17c29b3c2f25436f055c78 - stTrie.Commit(nil) // Root: 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67 - - accTrie, _ := trie.NewSecure(common.Hash{}, triedb) - acc := &Account{Balance: big.NewInt(1), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()} - val, _ := rlp.EncodeToBytes(acc) - accTrie.Update([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e + var helper = newHelper() + stRoot := helper.makeStorageTrie(common.Hash{}, common.Hash{}, []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, false) - acc = &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()} - val, _ = rlp.EncodeToBytes(acc) - accTrie.Update([]byte("acc-2"), val) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7 + helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) + helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) + helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: emptyCode.Bytes()}) - acc = &Account{Balance: big.NewInt(3), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()} - val, _ = rlp.EncodeToBytes(acc) - accTrie.Update([]byte("acc-3"), val) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2 - root, _, _ := accTrie.Commit(nil) // Root: 0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd - triedb.Commit(root, false, nil) + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) + root, snap := helper.CommitAndGenerate() if have, want := root, common.HexToHash("0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd"); have != want { t.Fatalf("have %#x want %#x", have, want) } - snap := generateSnapshot(diskdb, triedb, 16, root) select { case <-snap.genPending: // Snapshot generation succeeded @@ -75,63 +68,34 @@ func TestGeneration(t *testing.T) { t.Errorf("Snapshot generation failed") } checkSnapRoot(t, snap, root) + // Signal abortion to the generator and wait for it to tear down stop := make(chan *generatorStats) snap.genAbort <- stop <-stop } -func hashData(input []byte) common.Hash { - var hasher = sha3.NewLegacyKeccak256() - var hash common.Hash - hasher.Reset() - hasher.Write(input) - hasher.Sum(hash[:0]) - return hash -} - // Tests that snapshot generation with existent flat state. func TestGenerateExistentState(t *testing.T) { // We can't use statedb to make a test trie (circular dependency), so make // a fake one manually. We're going with a small account trie of 3 accounts, // two of which also has the same 3-slot storage trie attached. - var ( - diskdb = memorydb.New() - triedb = trie.NewDatabase(diskdb) - ) - stTrie, _ := trie.NewSecure(common.Hash{}, triedb) - stTrie.Update([]byte("key-1"), []byte("val-1")) // 0x1314700b81afc49f94db3623ef1df38f3ed18b73a1b7ea2f6c095118cf6118a0 - stTrie.Update([]byte("key-2"), []byte("val-2")) // 0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371 - stTrie.Update([]byte("key-3"), []byte("val-3")) // 0x51c71a47af0695957647fb68766d0becee77e953df17c29b3c2f25436f055c78 - stTrie.Commit(nil) // Root: 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67 - - accTrie, _ := trie.NewSecure(common.Hash{}, triedb) - acc := &Account{Balance: big.NewInt(1), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()} - val, _ := rlp.EncodeToBytes(acc) - accTrie.Update([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e - rawdb.WriteAccountSnapshot(diskdb, hashData([]byte("acc-1")), val) - rawdb.WriteStorageSnapshot(diskdb, hashData([]byte("acc-1")), hashData([]byte("key-1")), []byte("val-1")) - rawdb.WriteStorageSnapshot(diskdb, hashData([]byte("acc-1")), hashData([]byte("key-2")), []byte("val-2")) - rawdb.WriteStorageSnapshot(diskdb, hashData([]byte("acc-1")), hashData([]byte("key-3")), []byte("val-3")) - - acc = &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()} - val, _ = rlp.EncodeToBytes(acc) - accTrie.Update([]byte("acc-2"), val) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7 - diskdb.Put(hashData([]byte("acc-2")).Bytes(), val) - rawdb.WriteAccountSnapshot(diskdb, hashData([]byte("acc-2")), val) - - acc = &Account{Balance: big.NewInt(3), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()} - val, _ = rlp.EncodeToBytes(acc) - accTrie.Update([]byte("acc-3"), val) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2 - rawdb.WriteAccountSnapshot(diskdb, hashData([]byte("acc-3")), val) - rawdb.WriteStorageSnapshot(diskdb, hashData([]byte("acc-3")), hashData([]byte("key-1")), []byte("val-1")) - rawdb.WriteStorageSnapshot(diskdb, hashData([]byte("acc-3")), hashData([]byte("key-2")), []byte("val-2")) - rawdb.WriteStorageSnapshot(diskdb, hashData([]byte("acc-3")), hashData([]byte("key-3")), []byte("val-3")) - - root, _, _ := accTrie.Commit(nil) // Root: 0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd - triedb.Commit(root, false, nil) - - snap := generateSnapshot(diskdb, triedb, 16, root) + var helper = newHelper() + + stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) + helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) + helper.addSnapAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) + helper.addSnapStorage("acc-1", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) + + helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) + helper.addSnapAccount("acc-2", &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) + + stRoot = helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) + helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: emptyCode.Bytes()}) + helper.addSnapAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: emptyCode.Bytes()}) + helper.addSnapStorage("acc-3", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) + + root, snap := helper.CommitAndGenerate() select { case <-snap.genPending: // Snapshot generation succeeded @@ -140,6 +104,7 @@ func TestGenerateExistentState(t *testing.T) { t.Errorf("Snapshot generation failed") } checkSnapRoot(t, snap, root) + // Signal abortion to the generator and wait for it to tear down stop := make(chan *generatorStats) snap.genAbort <- stop @@ -163,7 +128,6 @@ func checkSnapRoot(t *testing.T, snap *diskLayer, trieRoot common.Hash) { } return hash, nil }, newGenerateStats(), true) - if err != nil { t.Fatal(err) } @@ -171,24 +135,26 @@ func checkSnapRoot(t *testing.T, snap *diskLayer, trieRoot common.Hash) { t.Fatalf("snaproot: %#x != trieroot #%x", snapRoot, trieRoot) } if err := CheckDanglingStorage(snap.diskdb); err != nil { - t.Fatalf("Detected dangling storages %v", err) + t.Fatalf("Detected dangling storages: %v", err) } } type testHelper struct { - diskdb *memorydb.Database + diskdb ethdb.Database triedb *trie.Database - accTrie *trie.SecureTrie + accTrie *trie.StateTrie + nodes *trie.MergedNodeSet } func newHelper() *testHelper { - diskdb := memorydb.New() + diskdb := rawdb.NewMemoryDatabase() triedb := trie.NewDatabase(diskdb) - accTrie, _ := trie.NewSecure(common.Hash{}, triedb) + accTrie, _ := trie.NewStateTrie(common.Hash{}, common.Hash{}, triedb) return &testHelper{ diskdb: diskdb, triedb: triedb, accTrie: accTrie, + nodes: trie.NewMergedNodeSet(), } } @@ -215,18 +181,33 @@ func (t *testHelper) addSnapStorage(accKey string, keys []string, vals []string) } } -func (t *testHelper) makeStorageTrie(keys []string, vals []string) []byte { - stTrie, _ := trie.NewSecure(common.Hash{}, t.triedb) +func (t *testHelper) makeStorageTrie(stateRoot, owner common.Hash, keys []string, vals []string, commit bool) []byte { + stTrie, _ := trie.NewStateTrie(owner, common.Hash{}, t.triedb) for i, k := range keys { stTrie.Update([]byte(k), []byte(vals[i])) } - root, _, _ := stTrie.Commit(nil) + if !commit { + return stTrie.Hash().Bytes() + } + root, nodes, _ := stTrie.Commit(false) + if nodes != nil { + t.nodes.Merge(nodes) + } return root.Bytes() } -func (t *testHelper) Generate() (common.Hash, *diskLayer) { - root, _, _ := t.accTrie.Commit(nil) +func (t *testHelper) Commit() common.Hash { + root, nodes, _ := t.accTrie.Commit(true) + if nodes != nil { + t.nodes.Merge(nodes) + } + t.triedb.Update(t.nodes) t.triedb.Commit(root, false, nil) + return root +} + +func (t *testHelper) CommitAndGenerate() (common.Hash, *diskLayer) { + root := t.Commit() snap := generateSnapshot(t.diskdb, t.triedb, 16, root) return root, snap } @@ -249,26 +230,29 @@ func (t *testHelper) Generate() (common.Hash, *diskLayer) { // - extra slots in the end func TestGenerateExistentStateWithWrongStorage(t *testing.T) { helper := newHelper() - stRoot := helper.makeStorageTrie([]string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) // Account one, empty root but non-empty database helper.addAccount("acc-1", &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) helper.addSnapStorage("acc-1", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) // Account two, non empty root but empty database + stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-2")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addAccount("acc-2", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) // Miss slots { // Account three, non empty root but misses slots in the beginning + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addAccount("acc-3", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) helper.addSnapStorage("acc-3", []string{"key-2", "key-3"}, []string{"val-2", "val-3"}) // Account four, non empty root but misses slots in the middle + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-4")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addAccount("acc-4", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) helper.addSnapStorage("acc-4", []string{"key-1", "key-3"}, []string{"val-1", "val-3"}) // Account five, non empty root but misses slots in the end + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-5")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addAccount("acc-5", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) helper.addSnapStorage("acc-5", []string{"key-1", "key-2"}, []string{"val-1", "val-2"}) } @@ -276,18 +260,22 @@ func TestGenerateExistentStateWithWrongStorage(t *testing.T) { // Wrong storage slots { // Account six, non empty root but wrong slots in the beginning + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-6")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addAccount("acc-6", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) helper.addSnapStorage("acc-6", []string{"key-1", "key-2", "key-3"}, []string{"badval-1", "val-2", "val-3"}) // Account seven, non empty root but wrong slots in the middle + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-7")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addAccount("acc-7", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) helper.addSnapStorage("acc-7", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "badval-2", "val-3"}) // Account eight, non empty root but wrong slots in the end + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-8")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addAccount("acc-8", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) helper.addSnapStorage("acc-8", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "badval-3"}) // Account 9, non empty root but rotated slots + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-9")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addAccount("acc-9", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) helper.addSnapStorage("acc-9", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-3", "val-2"}) } @@ -295,19 +283,22 @@ func TestGenerateExistentStateWithWrongStorage(t *testing.T) { // Extra storage slots { // Account 10, non empty root but extra slots in the beginning + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-10")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addAccount("acc-10", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) helper.addSnapStorage("acc-10", []string{"key-0", "key-1", "key-2", "key-3"}, []string{"val-0", "val-1", "val-2", "val-3"}) // Account 11, non empty root but extra slots in the middle + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-11")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addAccount("acc-11", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) helper.addSnapStorage("acc-11", []string{"key-1", "key-2", "key-2-1", "key-3"}, []string{"val-1", "val-2", "val-2-1", "val-3"}) // Account 12, non empty root but extra slots in the end + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-12")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addAccount("acc-12", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) helper.addSnapStorage("acc-12", []string{"key-1", "key-2", "key-3", "key-4"}, []string{"val-1", "val-2", "val-3", "val-4"}) } - root, snap := helper.Generate() + root, snap := helper.CommitAndGenerate() t.Logf("Root: %#x\n", root) // Root = 0x8746cce9fd9c658b2cfd639878ed6584b7a2b3e73bb40f607fcfa156002429a0 select { @@ -331,7 +322,12 @@ func TestGenerateExistentStateWithWrongStorage(t *testing.T) { // - extra accounts func TestGenerateExistentStateWithWrongAccounts(t *testing.T) { helper := newHelper() - stRoot := helper.makeStorageTrie([]string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) + + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-2")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-4")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) + stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-6")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) // Trie accounts [acc-1, acc-2, acc-3, acc-4, acc-6] // Extra accounts [acc-0, acc-5, acc-7] @@ -359,7 +355,7 @@ func TestGenerateExistentStateWithWrongAccounts(t *testing.T) { helper.addSnapAccount("acc-7", &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyRoot.Bytes()}) // after the end } - root, snap := helper.Generate() + root, snap := helper.CommitAndGenerate() t.Logf("Root: %#x\n", root) // Root = 0x825891472281463511e7ebcc7f109e4f9200c20fa384754e11fd605cd98464e8 select { @@ -383,29 +379,19 @@ func TestGenerateCorruptAccountTrie(t *testing.T) { // We can't use statedb to make a test trie (circular dependency), so make // a fake one manually. We're going with a small account trie of 3 accounts, // without any storage slots to keep the test smaller. - var ( - diskdb = memorydb.New() - triedb = trie.NewDatabase(diskdb) - ) - tr, _ := trie.NewSecure(common.Hash{}, triedb) - acc := &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()} - val, _ := rlp.EncodeToBytes(acc) - tr.Update([]byte("acc-1"), val) // 0xc7a30f39aff471c95d8a837497ad0e49b65be475cc0953540f80cfcdbdcd9074 + helper := newHelper() - acc = &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()} - val, _ = rlp.EncodeToBytes(acc) - tr.Update([]byte("acc-2"), val) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7 + helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) // 0xc7a30f39aff471c95d8a837497ad0e49b65be475cc0953540f80cfcdbdcd9074 + helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7 + helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) // 0x19ead688e907b0fab07176120dceec244a72aff2f0aa51e8b827584e378772f4 - acc = &Account{Balance: big.NewInt(3), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()} - val, _ = rlp.EncodeToBytes(acc) - tr.Update([]byte("acc-3"), val) // 0x19ead688e907b0fab07176120dceec244a72aff2f0aa51e8b827584e378772f4 - tr.Commit(nil) // Root: 0xa04693ea110a31037fb5ee814308a6f1d76bdab0b11676bdf4541d2de55ba978 + root := helper.Commit() // Root: 0xa04693ea110a31037fb5ee814308a6f1d76bdab0b11676bdf4541d2de55ba978 // Delete an account trie leaf and ensure the generator chokes - triedb.Commit(common.HexToHash("0xa04693ea110a31037fb5ee814308a6f1d76bdab0b11676bdf4541d2de55ba978"), false, nil) - diskdb.Delete(common.HexToHash("0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7").Bytes()) + helper.triedb.Commit(root, false, nil) + helper.diskdb.Delete(common.HexToHash("0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7").Bytes()) - snap := generateSnapshot(diskdb, triedb, 16, common.HexToHash("0xa04693ea110a31037fb5ee814308a6f1d76bdab0b11676bdf4541d2de55ba978")) + snap := generateSnapshot(helper.diskdb, helper.triedb, 16, root) select { case <-snap.genPending: // Snapshot generation succeeded @@ -427,45 +413,20 @@ func TestGenerateMissingStorageTrie(t *testing.T) { // We can't use statedb to make a test trie (circular dependency), so make // a fake one manually. We're going with a small account trie of 3 accounts, // two of which also has the same 3-slot storage trie attached. - var ( - diskdb = memorydb.New() - triedb = trie.NewDatabase(diskdb) - ) - stTrie, _ := trie.NewSecure(common.Hash{}, triedb) - stTrie.Update([]byte("key-1"), []byte("val-1")) // 0x1314700b81afc49f94db3623ef1df38f3ed18b73a1b7ea2f6c095118cf6118a0 - stTrie.Update([]byte("key-2"), []byte("val-2")) // 0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371 - stTrie.Update([]byte("key-3"), []byte("val-3")) // 0x51c71a47af0695957647fb68766d0becee77e953df17c29b3c2f25436f055c78 - stTrie.Commit(nil) // Root: 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67 - - accTrie, _ := trie.NewSecure(common.Hash{}, triedb) - acc := &Account{Balance: big.NewInt(1), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()} - val, _ := rlp.EncodeToBytes(acc) - accTrie.Update([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e - - acc = &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()} - val, _ = rlp.EncodeToBytes(acc) - accTrie.Update([]byte("acc-2"), val) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7 - - acc = &Account{Balance: big.NewInt(3), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()} - val, _ = rlp.EncodeToBytes(acc) - accTrie.Update([]byte("acc-3"), val) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2 - accTrie.Commit(nil) // Root: 0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd - - // We can only corrupt the disk database, so flush the tries out - triedb.Reference( - common.HexToHash("0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67"), - common.HexToHash("0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e"), - ) - triedb.Reference( - common.HexToHash("0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67"), - common.HexToHash("0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2"), - ) - triedb.Commit(common.HexToHash("0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd"), false, nil) + helper := newHelper() + + stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) // 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67 + helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e + helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7 + stRoot = helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) + helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: emptyCode.Bytes()}) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2 + + root := helper.Commit() // Delete a storage trie root and ensure the generator chokes - diskdb.Delete(common.HexToHash("0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67").Bytes()) + helper.diskdb.Delete(stRoot) - snap := generateSnapshot(diskdb, triedb, 16, common.HexToHash("0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd")) + snap := generateSnapshot(helper.diskdb, helper.triedb, 16, root) select { case <-snap.genPending: // Snapshot generation succeeded @@ -486,45 +447,20 @@ func TestGenerateCorruptStorageTrie(t *testing.T) { // We can't use statedb to make a test trie (circular dependency), so make // a fake one manually. We're going with a small account trie of 3 accounts, // two of which also has the same 3-slot storage trie attached. - var ( - diskdb = memorydb.New() - triedb = trie.NewDatabase(diskdb) - ) - stTrie, _ := trie.NewSecure(common.Hash{}, triedb) - stTrie.Update([]byte("key-1"), []byte("val-1")) // 0x1314700b81afc49f94db3623ef1df38f3ed18b73a1b7ea2f6c095118cf6118a0 - stTrie.Update([]byte("key-2"), []byte("val-2")) // 0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371 - stTrie.Update([]byte("key-3"), []byte("val-3")) // 0x51c71a47af0695957647fb68766d0becee77e953df17c29b3c2f25436f055c78 - stTrie.Commit(nil) // Root: 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67 - - accTrie, _ := trie.NewSecure(common.Hash{}, triedb) - acc := &Account{Balance: big.NewInt(1), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()} - val, _ := rlp.EncodeToBytes(acc) - accTrie.Update([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e - - acc = &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()} - val, _ = rlp.EncodeToBytes(acc) - accTrie.Update([]byte("acc-2"), val) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7 - - acc = &Account{Balance: big.NewInt(3), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()} - val, _ = rlp.EncodeToBytes(acc) - accTrie.Update([]byte("acc-3"), val) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2 - accTrie.Commit(nil) // Root: 0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd - - // We can only corrupt the disk database, so flush the tries out - triedb.Reference( - common.HexToHash("0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67"), - common.HexToHash("0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e"), - ) - triedb.Reference( - common.HexToHash("0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67"), - common.HexToHash("0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2"), - ) - triedb.Commit(common.HexToHash("0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd"), false, nil) + helper := newHelper() + + stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) // 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67 + helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e + helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7 + stRoot = helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) + helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: emptyCode.Bytes()}) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2 + + root := helper.Commit() // Delete a storage trie leaf and ensure the generator chokes - diskdb.Delete(common.HexToHash("0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371").Bytes()) + helper.diskdb.Delete(common.HexToHash("0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371").Bytes()) - snap := generateSnapshot(diskdb, triedb, 16, common.HexToHash("0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd")) + snap := generateSnapshot(helper.diskdb, helper.triedb, 16, root) select { case <-snap.genPending: // Snapshot generation succeeded @@ -539,56 +475,51 @@ func TestGenerateCorruptStorageTrie(t *testing.T) { <-stop } -func getStorageTrie(n int, triedb *trie.Database) *trie.SecureTrie { - stTrie, _ := trie.NewSecure(common.Hash{}, triedb) - for i := 0; i < n; i++ { - k := fmt.Sprintf("key-%d", i) - v := fmt.Sprintf("val-%d", i) - stTrie.Update([]byte(k), []byte(v)) - } - stTrie.Commit(nil) - return stTrie -} - // Tests that snapshot generation when an extra account with storage exists in the snap state. func TestGenerateWithExtraAccounts(t *testing.T) { - var ( - diskdb = memorydb.New() - triedb = trie.NewDatabase(diskdb) - stTrie = getStorageTrie(5, triedb) - ) - accTrie, _ := trie.NewSecure(common.Hash{}, triedb) - { // Account one in the trie - acc := &Account{Balance: big.NewInt(1), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()} + helper := newHelper() + { + // Account one in the trie + stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), + []string{"key-1", "key-2", "key-3", "key-4", "key-5"}, + []string{"val-1", "val-2", "val-3", "val-4", "val-5"}, + true, + ) + acc := &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()} val, _ := rlp.EncodeToBytes(acc) - accTrie.Update([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e + helper.accTrie.Update([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e + // Identical in the snap key := hashData([]byte("acc-1")) - rawdb.WriteAccountSnapshot(diskdb, key, val) - rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("key-1")), []byte("val-1")) - rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("key-2")), []byte("val-2")) - rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("key-3")), []byte("val-3")) - rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("key-4")), []byte("val-4")) - rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("key-5")), []byte("val-5")) - } - { // Account two exists only in the snapshot - acc := &Account{Balance: big.NewInt(1), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()} + rawdb.WriteAccountSnapshot(helper.triedb.DiskDB(), key, val) + rawdb.WriteStorageSnapshot(helper.triedb.DiskDB(), key, hashData([]byte("key-1")), []byte("val-1")) + rawdb.WriteStorageSnapshot(helper.triedb.DiskDB(), key, hashData([]byte("key-2")), []byte("val-2")) + rawdb.WriteStorageSnapshot(helper.triedb.DiskDB(), key, hashData([]byte("key-3")), []byte("val-3")) + rawdb.WriteStorageSnapshot(helper.triedb.DiskDB(), key, hashData([]byte("key-4")), []byte("val-4")) + rawdb.WriteStorageSnapshot(helper.triedb.DiskDB(), key, hashData([]byte("key-5")), []byte("val-5")) + } + { + // Account two exists only in the snapshot + stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-2")), + []string{"key-1", "key-2", "key-3", "key-4", "key-5"}, + []string{"val-1", "val-2", "val-3", "val-4", "val-5"}, + true, + ) + acc := &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()} val, _ := rlp.EncodeToBytes(acc) key := hashData([]byte("acc-2")) - rawdb.WriteAccountSnapshot(diskdb, key, val) - rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("b-key-1")), []byte("b-val-1")) - rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("b-key-2")), []byte("b-val-2")) - rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("b-key-3")), []byte("b-val-3")) - } - root, _, _ := accTrie.Commit(nil) - t.Logf("root: %x", root) - triedb.Commit(root, false, nil) + rawdb.WriteAccountSnapshot(helper.triedb.DiskDB(), key, val) + rawdb.WriteStorageSnapshot(helper.triedb.DiskDB(), key, hashData([]byte("b-key-1")), []byte("b-val-1")) + rawdb.WriteStorageSnapshot(helper.triedb.DiskDB(), key, hashData([]byte("b-key-2")), []byte("b-val-2")) + rawdb.WriteStorageSnapshot(helper.triedb.DiskDB(), key, hashData([]byte("b-key-3")), []byte("b-val-3")) + } + root := helper.Commit() + // To verify the test: If we now inspect the snap db, there should exist extraneous storage items - if data := rawdb.ReadStorageSnapshot(diskdb, hashData([]byte("acc-2")), hashData([]byte("b-key-1"))); data == nil { + if data := rawdb.ReadStorageSnapshot(helper.triedb.DiskDB(), hashData([]byte("acc-2")), hashData([]byte("b-key-1"))); data == nil { t.Fatalf("expected snap storage to exist") } - - snap := generateSnapshot(diskdb, triedb, 16, root) + snap := generateSnapshot(helper.diskdb, helper.triedb, 16, root) select { case <-snap.genPending: // Snapshot generation succeeded @@ -597,12 +528,13 @@ func TestGenerateWithExtraAccounts(t *testing.T) { t.Errorf("Snapshot generation failed") } checkSnapRoot(t, snap, root) + // Signal abortion to the generator and wait for it to tear down stop := make(chan *generatorStats) snap.genAbort <- stop <-stop // If we now inspect the snap db, there should exist no extraneous storage items - if data := rawdb.ReadStorageSnapshot(diskdb, hashData([]byte("acc-2")), hashData([]byte("b-key-1"))); data != nil { + if data := rawdb.ReadStorageSnapshot(helper.triedb.DiskDB(), hashData([]byte("acc-2")), hashData([]byte("b-key-1"))); data != nil { t.Fatalf("expected slot to be removed, got %v", string(data)) } } @@ -616,37 +548,36 @@ func TestGenerateWithManyExtraAccounts(t *testing.T) { if false { enableLogging() } - var ( - diskdb = memorydb.New() - triedb = trie.NewDatabase(diskdb) - stTrie = getStorageTrie(3, triedb) - ) - accTrie, _ := trie.NewSecure(common.Hash{}, triedb) - { // Account one in the trie - acc := &Account{Balance: big.NewInt(1), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()} + helper := newHelper() + { + // Account one in the trie + stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), + []string{"key-1", "key-2", "key-3"}, + []string{"val-1", "val-2", "val-3"}, + true, + ) + acc := &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()} val, _ := rlp.EncodeToBytes(acc) - accTrie.Update([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e + helper.accTrie.Update([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e + // Identical in the snap key := hashData([]byte("acc-1")) - rawdb.WriteAccountSnapshot(diskdb, key, val) - rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("key-1")), []byte("val-1")) - rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("key-2")), []byte("val-2")) - rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("key-3")), []byte("val-3")) + rawdb.WriteAccountSnapshot(helper.diskdb, key, val) + rawdb.WriteStorageSnapshot(helper.diskdb, key, hashData([]byte("key-1")), []byte("val-1")) + rawdb.WriteStorageSnapshot(helper.diskdb, key, hashData([]byte("key-2")), []byte("val-2")) + rawdb.WriteStorageSnapshot(helper.diskdb, key, hashData([]byte("key-3")), []byte("val-3")) } - { // 100 accounts exist only in snapshot + { + // 100 accounts exist only in snapshot for i := 0; i < 1000; i++ { //acc := &Account{Balance: big.NewInt(int64(i)), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()} acc := &Account{Balance: big.NewInt(int64(i)), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()} val, _ := rlp.EncodeToBytes(acc) key := hashData([]byte(fmt.Sprintf("acc-%d", i))) - rawdb.WriteAccountSnapshot(diskdb, key, val) + rawdb.WriteAccountSnapshot(helper.diskdb, key, val) } } - root, _, _ := accTrie.Commit(nil) - t.Logf("root: %x", root) - triedb.Commit(root, false, nil) - - snap := generateSnapshot(diskdb, triedb, 16, root) + root, snap := helper.CommitAndGenerate() select { case <-snap.genPending: // Snapshot generation succeeded @@ -675,31 +606,22 @@ func TestGenerateWithExtraBeforeAndAfter(t *testing.T) { if false { enableLogging() } - var ( - diskdb = memorydb.New() - triedb = trie.NewDatabase(diskdb) - ) - accTrie, _ := trie.New(common.Hash{}, triedb) + helper := newHelper() { acc := &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()} val, _ := rlp.EncodeToBytes(acc) - accTrie.Update(common.HexToHash("0x03").Bytes(), val) - accTrie.Update(common.HexToHash("0x07").Bytes(), val) - - rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x01"), val) - rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x02"), val) - rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x03"), val) - rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x04"), val) - rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x05"), val) - rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x06"), val) - rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x07"), val) - } - - root, _, _ := accTrie.Commit(nil) - t.Logf("root: %x", root) - triedb.Commit(root, false, nil) - - snap := generateSnapshot(diskdb, triedb, 16, root) + helper.accTrie.Update(common.HexToHash("0x03").Bytes(), val) + helper.accTrie.Update(common.HexToHash("0x07").Bytes(), val) + + rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x01"), val) + rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x02"), val) + rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x03"), val) + rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x04"), val) + rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x05"), val) + rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x06"), val) + rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x07"), val) + } + root, snap := helper.CommitAndGenerate() select { case <-snap.genPending: // Snapshot generation succeeded @@ -721,29 +643,20 @@ func TestGenerateWithMalformedSnapdata(t *testing.T) { if false { enableLogging() } - var ( - diskdb = memorydb.New() - triedb = trie.NewDatabase(diskdb) - ) - accTrie, _ := trie.New(common.Hash{}, triedb) + helper := newHelper() { acc := &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()} val, _ := rlp.EncodeToBytes(acc) - accTrie.Update(common.HexToHash("0x03").Bytes(), val) + helper.accTrie.Update(common.HexToHash("0x03").Bytes(), val) junk := make([]byte, 100) copy(junk, []byte{0xde, 0xad}) - rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x02"), junk) - rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x03"), junk) - rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x04"), junk) - rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x05"), junk) + rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x02"), junk) + rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x03"), junk) + rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x04"), junk) + rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x05"), junk) } - - root, _, _ := accTrie.Commit(nil) - t.Logf("root: %x", root) - triedb.Commit(root, false, nil) - - snap := generateSnapshot(diskdb, triedb, 16, root) + root, snap := helper.CommitAndGenerate() select { case <-snap.genPending: // Snapshot generation succeeded @@ -757,7 +670,7 @@ func TestGenerateWithMalformedSnapdata(t *testing.T) { snap.genAbort <- stop <-stop // If we now inspect the snap db, there should exist no extraneous storage items - if data := rawdb.ReadStorageSnapshot(diskdb, hashData([]byte("acc-2")), hashData([]byte("b-key-1"))); data != nil { + if data := rawdb.ReadStorageSnapshot(helper.diskdb, hashData([]byte("acc-2")), hashData([]byte("b-key-1"))); data != nil { t.Fatalf("expected slot to be removed, got %v", string(data)) } } @@ -767,13 +680,13 @@ func TestGenerateFromEmptySnap(t *testing.T) { accountCheckRange = 10 storageCheckRange = 20 helper := newHelper() - stRoot := helper.makeStorageTrie([]string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) // Add 1K accounts to the trie for i := 0; i < 400; i++ { + stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte(fmt.Sprintf("acc-%d", i))), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addTrieAccount(fmt.Sprintf("acc-%d", i), &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) } - root, snap := helper.Generate() + root, snap := helper.CommitAndGenerate() t.Logf("Root: %#x\n", root) // Root: 0x6f7af6d2e1a1bf2b84a3beb3f8b64388465fbc1e274ca5d5d3fc787ca78f59e4 select { @@ -802,12 +715,12 @@ func TestGenerateWithIncompleteStorage(t *testing.T) { helper := newHelper() stKeys := []string{"1", "2", "3", "4", "5", "6", "7", "8"} stVals := []string{"v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8"} - stRoot := helper.makeStorageTrie(stKeys, stVals) // We add 8 accounts, each one is missing exactly one of the storage slots. This means // we don't have to order the keys and figure out exactly which hash-key winds up // on the sensitive spots at the boundaries for i := 0; i < 8; i++ { accKey := fmt.Sprintf("acc-%d", i) + stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte(accKey)), stKeys, stVals, true) helper.addAccount(accKey, &Account{Balance: big.NewInt(int64(i)), Root: stRoot, CodeHash: emptyCode.Bytes()}) var moddedKeys []string var moddedVals []string @@ -819,8 +732,7 @@ func TestGenerateWithIncompleteStorage(t *testing.T) { } helper.addSnapStorage(accKey, moddedKeys, moddedVals) } - - root, snap := helper.Generate() + root, snap := helper.CommitAndGenerate() t.Logf("Root: %#x\n", root) // Root: 0xca73f6f05ba4ca3024ef340ef3dfca8fdabc1b677ff13f5a9571fd49c16e67ff select { @@ -899,10 +811,12 @@ func populateDangling(disk ethdb.KeyValueStore) { // This test will populate some dangling storages to see if they can be cleaned up. func TestGenerateCompleteSnapshotWithDanglingStorage(t *testing.T) { var helper = newHelper() - stRoot := helper.makeStorageTrie([]string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) + stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) helper.addAccount("acc-2", &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) + + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addAccount("acc-3", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) helper.addSnapStorage("acc-1", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) @@ -910,7 +824,7 @@ func TestGenerateCompleteSnapshotWithDanglingStorage(t *testing.T) { populateDangling(helper.diskdb) - root, snap := helper.Generate() + root, snap := helper.CommitAndGenerate() select { case <-snap.genPending: // Snapshot generation succeeded @@ -932,15 +846,17 @@ func TestGenerateCompleteSnapshotWithDanglingStorage(t *testing.T) { // This test will populate some dangling storages to see if they can be cleaned up. func TestGenerateBrokenSnapshotWithDanglingStorage(t *testing.T) { var helper = newHelper() - stRoot := helper.makeStorageTrie([]string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) + stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) + + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: emptyCode.Bytes()}) populateDangling(helper.diskdb) - root, snap := helper.Generate() + root, snap := helper.CommitAndGenerate() select { case <-snap.genPending: // Snapshot generation succeeded diff --git a/core/state/snapshot/iterator_fast.go b/core/state/snapshot/iterator_fast.go index 48069b8fcf5c0..435c28e96f9e8 100644 --- a/core/state/snapshot/iterator_fast.go +++ b/core/state/snapshot/iterator_fast.go @@ -319,7 +319,7 @@ func (fi *fastIterator) Slot() []byte { } // Release iterates over all the remaining live layer iterators and releases each -// of thme individually. +// of them individually. func (fi *fastIterator) Release() { for _, it := range fi.iterators { it.it.Release() @@ -327,7 +327,7 @@ func (fi *fastIterator) Release() { fi.iterators = nil } -// Debug is a convencience helper during testing +// Debug is a convenience helper during testing func (fi *fastIterator) Debug() { for _, it := range fi.iterators { fmt.Printf("[p=%v v=%v] ", it.priority, it.it.Hash()[0]) diff --git a/core/state/snapshot/journal.go b/core/state/snapshot/journal.go index 6836a574090cd..9a22f27373068 100644 --- a/core/state/snapshot/journal.go +++ b/core/state/snapshot/journal.go @@ -108,44 +108,15 @@ func loadAndParseJournal(db ethdb.KeyValueStore, base *diskLayer) (snapshot, jou // So if there is no journal, or the journal is invalid(e.g. the journal // is not matched with disk layer; or the it's the legacy-format journal, // etc.), we just discard all diffs and try to recover them later. - journal := rawdb.ReadSnapshotJournal(db) - if len(journal) == 0 { - log.Warn("Loaded snapshot journal", "diskroot", base.root, "diffs", "missing") - return base, generator, nil - } - r := rlp.NewStream(bytes.NewReader(journal), 0) - - // Firstly, resolve the first element as the journal version - version, err := r.Uint() + var current snapshot = base + err := iterateJournal(db, func(parent common.Hash, root common.Hash, destructSet map[common.Hash]struct{}, accountData map[common.Hash][]byte, storageData map[common.Hash]map[common.Hash][]byte) error { + current = newDiffLayer(current, root, destructSet, accountData, storageData) + return nil + }) if err != nil { - log.Warn("Failed to resolve the journal version", "error", err) - return base, generator, nil - } - if version != journalVersion { - log.Warn("Discarded the snapshot journal with wrong version", "required", journalVersion, "got", version) - return base, generator, nil - } - // Secondly, resolve the disk layer root, ensure it's continuous - // with disk layer. Note now we can ensure it's the snapshot journal - // correct version, so we expect everything can be resolved properly. - var root common.Hash - if err := r.Decode(&root); err != nil { - return nil, journalGenerator{}, errors.New("missing disk layer root") - } - // The diff journal is not matched with disk, discard them. - // It can happen that Geth crashes without persisting the latest - // diff journal. - if !bytes.Equal(root.Bytes(), base.root.Bytes()) { - log.Warn("Loaded snapshot journal", "diskroot", base.root, "diffs", "unmatched") return base, generator, nil } - // Load all the snapshot diffs from the journal - snapshot, err := loadDiffLayer(base, r) - if err != nil { - return nil, journalGenerator{}, err - } - log.Debug("Loaded snapshot journal", "diskroot", base.root, "diffhead", snapshot.Root()) - return snapshot, generator, nil + return current, generator, nil } // loadSnapshot loads a pre-existing state snapshot backed by a key-value store. @@ -218,57 +189,6 @@ func loadSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, return snapshot, false, nil } -// loadDiffLayer reads the next sections of a snapshot journal, reconstructing a new -// diff and verifying that it can be linked to the requested parent. -func loadDiffLayer(parent snapshot, r *rlp.Stream) (snapshot, error) { - // Read the next diff journal entry - var root common.Hash - if err := r.Decode(&root); err != nil { - // The first read may fail with EOF, marking the end of the journal - if err == io.EOF { - return parent, nil - } - return nil, fmt.Errorf("load diff root: %v", err) - } - var destructs []journalDestruct - if err := r.Decode(&destructs); err != nil { - return nil, fmt.Errorf("load diff destructs: %v", err) - } - destructSet := make(map[common.Hash]struct{}) - for _, entry := range destructs { - destructSet[entry.Hash] = struct{}{} - } - var accounts []journalAccount - if err := r.Decode(&accounts); err != nil { - return nil, fmt.Errorf("load diff accounts: %v", err) - } - accountData := make(map[common.Hash][]byte) - for _, entry := range accounts { - if len(entry.Blob) > 0 { // RLP loses nil-ness, but `[]byte{}` is not a valid item, so reinterpret that - accountData[entry.Hash] = entry.Blob - } else { - accountData[entry.Hash] = nil - } - } - var storage []journalStorage - if err := r.Decode(&storage); err != nil { - return nil, fmt.Errorf("load diff storage: %v", err) - } - storageData := make(map[common.Hash]map[common.Hash][]byte) - for _, entry := range storage { - slots := make(map[common.Hash][]byte) - for i, key := range entry.Keys { - if len(entry.Vals[i]) > 0 { // RLP loses nil-ness, but `[]byte{}` is not a valid item, so reinterpret that - slots[key] = entry.Vals[i] - } else { - slots[key] = nil - } - } - storageData[entry.Hash] = slots - } - return loadDiffLayer(newDiffLayer(parent, root, destructSet, accountData, storageData), r) -} - // Journal terminates any in-progress snapshot generation, also implicitly pushing // the progress into the database. func (dl *diskLayer) Journal(buffer *bytes.Buffer) (common.Hash, error) { @@ -345,3 +265,96 @@ func (dl *diffLayer) Journal(buffer *bytes.Buffer) (common.Hash, error) { log.Debug("Journalled diff layer", "root", dl.root, "parent", dl.parent.Root()) return base, nil } + +// journalCallback is a function which is invoked by iterateJournal, every +// time a difflayer is loaded from disk. +type journalCallback = func(parent common.Hash, root common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) error + +// iterateJournal iterates through the journalled difflayers, loading them from +// the database, and invoking the callback for each loaded layer. +// The order is incremental; starting with the bottom-most difflayer, going towards +// the most recent layer. +// This method returns error either if there was some error reading from disk, +// OR if the callback returns an error when invoked. +func iterateJournal(db ethdb.KeyValueReader, callback journalCallback) error { + journal := rawdb.ReadSnapshotJournal(db) + if len(journal) == 0 { + log.Warn("Loaded snapshot journal", "diffs", "missing") + return nil + } + r := rlp.NewStream(bytes.NewReader(journal), 0) + // Firstly, resolve the first element as the journal version + version, err := r.Uint64() + if err != nil { + log.Warn("Failed to resolve the journal version", "error", err) + return errors.New("failed to resolve journal version") + } + if version != journalVersion { + log.Warn("Discarded the snapshot journal with wrong version", "required", journalVersion, "got", version) + return errors.New("wrong journal version") + } + // Secondly, resolve the disk layer root, ensure it's continuous + // with disk layer. Note now we can ensure it's the snapshot journal + // correct version, so we expect everything can be resolved properly. + var parent common.Hash + if err := r.Decode(&parent); err != nil { + return errors.New("missing disk layer root") + } + if baseRoot := rawdb.ReadSnapshotRoot(db); baseRoot != parent { + log.Warn("Loaded snapshot journal", "diskroot", baseRoot, "diffs", "unmatched") + return fmt.Errorf("mismatched disk and diff layers") + } + for { + var ( + root common.Hash + destructs []journalDestruct + accounts []journalAccount + storage []journalStorage + destructSet = make(map[common.Hash]struct{}) + accountData = make(map[common.Hash][]byte) + storageData = make(map[common.Hash]map[common.Hash][]byte) + ) + // Read the next diff journal entry + if err := r.Decode(&root); err != nil { + // The first read may fail with EOF, marking the end of the journal + if errors.Is(err, io.EOF) { + return nil + } + return fmt.Errorf("load diff root: %v", err) + } + if err := r.Decode(&destructs); err != nil { + return fmt.Errorf("load diff destructs: %v", err) + } + if err := r.Decode(&accounts); err != nil { + return fmt.Errorf("load diff accounts: %v", err) + } + if err := r.Decode(&storage); err != nil { + return fmt.Errorf("load diff storage: %v", err) + } + for _, entry := range destructs { + destructSet[entry.Hash] = struct{}{} + } + for _, entry := range accounts { + if len(entry.Blob) > 0 { // RLP loses nil-ness, but `[]byte{}` is not a valid item, so reinterpret that + accountData[entry.Hash] = entry.Blob + } else { + accountData[entry.Hash] = nil + } + } + for _, entry := range storage { + slots := make(map[common.Hash][]byte) + for i, key := range entry.Keys { + if len(entry.Vals[i]) > 0 { // RLP loses nil-ness, but `[]byte{}` is not a valid item, so reinterpret that + slots[key] = entry.Vals[i] + } else { + slots[key] = nil + } + } + storageData[entry.Hash] = slots + } + if err := callback(parent, root, destructSet, accountData, storageData); err != nil { + return err + } + parent = root + } +} diff --git a/core/state/snapshot/snapshot_test.go b/core/state/snapshot/snapshot_test.go index bc4e5cbd04620..7c8077b652ed7 100644 --- a/core/state/snapshot/snapshot_test.go +++ b/core/state/snapshot/snapshot_test.go @@ -265,7 +265,7 @@ func TestPostCapBasicDataAccess(t *testing.T) { snaps.Update(common.HexToHash("0xa3"), common.HexToHash("0xa2"), nil, setAccount("0xa3"), nil) snaps.Update(common.HexToHash("0xb3"), common.HexToHash("0xb2"), nil, setAccount("0xb3"), nil) - // checkExist verifies if an account exiss in a snapshot + // checkExist verifies if an account exists in a snapshot checkExist := func(layer *diffLayer, key string) error { if data, _ := layer.Account(common.HexToHash(key)); data == nil { return fmt.Errorf("expected %x to exist, got nil", common.HexToHash(key)) diff --git a/core/state/snapshot/dangling.go b/core/state/snapshot/utils.go similarity index 55% rename from core/state/snapshot/dangling.go rename to core/state/snapshot/utils.go index ca73da793f7a4..fa1f216e6826f 100644 --- a/core/state/snapshot/dangling.go +++ b/core/state/snapshot/utils.go @@ -18,9 +18,7 @@ package snapshot import ( "bytes" - "errors" "fmt" - "io" "time" "github.com/ethereum/go-ethereum/common" @@ -34,7 +32,7 @@ import ( // storage also has corresponding account data. func CheckDanglingStorage(chaindb ethdb.KeyValueStore) error { if err := checkDanglingDiskStorage(chaindb); err != nil { - return err + log.Error("Database check error", "err", err) } return checkDanglingMemStorage(chaindb) } @@ -75,81 +73,80 @@ func checkDanglingDiskStorage(chaindb ethdb.KeyValueStore) error { // checkDanglingMemStorage checks if there is any 'dangling' storage in the journalled // snapshot difflayers. func checkDanglingMemStorage(db ethdb.KeyValueStore) error { - var ( - start = time.Now() - journal = rawdb.ReadSnapshotJournal(db) - ) - if len(journal) == 0 { - log.Warn("Loaded snapshot journal", "diffs", "missing") + start := time.Now() + log.Info("Checking dangling journalled storage") + err := iterateJournal(db, func(pRoot, root common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) error { + for accHash := range storage { + if _, ok := accounts[accHash]; !ok { + log.Error("Dangling storage - missing account", "account", fmt.Sprintf("%#x", accHash), "root", root) + } + } return nil - } - r := rlp.NewStream(bytes.NewReader(journal), 0) - // Firstly, resolve the first element as the journal version - version, err := r.Uint() + }) if err != nil { - log.Warn("Failed to resolve the journal version", "error", err) - return nil - } - if version != journalVersion { - log.Warn("Discarded the snapshot journal with wrong version", "required", journalVersion, "got", version) - return nil - } - // Secondly, resolve the disk layer root, ensure it's continuous - // with disk layer. Note now we can ensure it's the snapshot journal - // correct version, so we expect everything can be resolved properly. - var root common.Hash - if err := r.Decode(&root); err != nil { - return errors.New("missing disk layer root") - } - // The diff journal is not matched with disk, discard them. - // It can happen that Geth crashes without persisting the latest - // diff journal. - // Load all the snapshot diffs from the journal - if err := checkDanglingJournalStorage(r); err != nil { + log.Info("Failed to resolve snapshot journal", "err", err) return err } log.Info("Verified the snapshot journalled storage", "time", common.PrettyDuration(time.Since(start))) return nil } -// loadDiffLayer reads the next sections of a snapshot journal, reconstructing a new -// diff and verifying that it can be linked to the requested parent. -func checkDanglingJournalStorage(r *rlp.Stream) error { - for { - // Read the next diff journal entry - var root common.Hash - if err := r.Decode(&root); err != nil { - // The first read may fail with EOF, marking the end of the journal - if err == io.EOF { - return nil - } - return fmt.Errorf("load diff root: %v", err) +// CheckJournalAccount shows information about an account, from the disk layer and +// up through the diff layers. +func CheckJournalAccount(db ethdb.KeyValueStore, hash common.Hash) error { + // Look up the disk layer first + baseRoot := rawdb.ReadSnapshotRoot(db) + fmt.Printf("Disklayer: Root: %x\n", baseRoot) + if data := rawdb.ReadAccountSnapshot(db, hash); data != nil { + account := new(Account) + if err := rlp.DecodeBytes(data, account); err != nil { + panic(err) } - var destructs []journalDestruct - if err := r.Decode(&destructs); err != nil { - return fmt.Errorf("load diff destructs: %v", err) + fmt.Printf("\taccount.nonce: %d\n", account.Nonce) + fmt.Printf("\taccount.balance: %x\n", account.Balance) + fmt.Printf("\taccount.root: %x\n", account.Root) + fmt.Printf("\taccount.codehash: %x\n", account.CodeHash) + } + // Check storage + { + it := rawdb.NewKeyLengthIterator(db.NewIterator(append(rawdb.SnapshotStoragePrefix, hash.Bytes()...), nil), 1+2*common.HashLength) + fmt.Printf("\tStorage:\n") + for it.Next() { + slot := it.Key()[33:] + fmt.Printf("\t\t%x: %x\n", slot, it.Value()) } - var accounts []journalAccount - if err := r.Decode(&accounts); err != nil { - return fmt.Errorf("load diff accounts: %v", err) + it.Release() + } + var depth = 0 + + return iterateJournal(db, func(pRoot, root common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) error { + _, a := accounts[hash] + _, b := destructs[hash] + _, c := storage[hash] + depth++ + if !a && !b && !c { + return nil } - accountData := make(map[common.Hash][]byte) - for _, entry := range accounts { - if len(entry.Blob) > 0 { // RLP loses nil-ness, but `[]byte{}` is not a valid item, so reinterpret that - accountData[entry.Hash] = entry.Blob - } else { - accountData[entry.Hash] = nil + fmt.Printf("Disklayer+%d: Root: %x, parent %x\n", depth, root, pRoot) + if data, ok := accounts[hash]; ok { + account := new(Account) + if err := rlp.DecodeBytes(data, account); err != nil { + panic(err) } + fmt.Printf("\taccount.nonce: %d\n", account.Nonce) + fmt.Printf("\taccount.balance: %x\n", account.Balance) + fmt.Printf("\taccount.root: %x\n", account.Root) + fmt.Printf("\taccount.codehash: %x\n", account.CodeHash) } - var storage []journalStorage - if err := r.Decode(&storage); err != nil { - return fmt.Errorf("load diff storage: %v", err) + if _, ok := destructs[hash]; ok { + fmt.Printf("\t Destructed!") } - for _, entry := range storage { - if _, ok := accountData[entry.Hash]; !ok { - log.Error("Dangling storage - missing account", "account", fmt.Sprintf("%#x", entry.Hash), "root", root) - return fmt.Errorf("dangling journal snapshot storage account %#x", entry.Hash) + if data, ok := storage[hash]; ok { + fmt.Printf("\tStorage\n") + for k, v := range data { + fmt.Printf("\t\t%x: %x\n", k, v) } } - } + return nil + }) } diff --git a/core/state/state_object.go b/core/state/state_object.go index bcb6dca4f56bc..a23df895458c1 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -28,6 +28,7 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie" ) var emptyCodeHash = crypto.Keccak256(nil) @@ -49,7 +50,7 @@ func (s Storage) String() (str string) { } func (s Storage) Copy() Storage { - cpy := make(Storage) + cpy := make(Storage, len(s)) for key, value := range s { cpy[key] = value } @@ -154,7 +155,7 @@ func (s *stateObject) getTrie(db Database) Trie { if s.data.Root != emptyRoot && s.db.prefetcher != nil { // When the miner is creating the pending state, there is no // prefetcher - s.trie = s.db.prefetcher.trie(s.data.Root) + s.trie = s.db.prefetcher.trie(s.addrHash, s.data.Root) } if s.trie == nil { var err error @@ -295,7 +296,7 @@ func (s *stateObject) finalise(prefetch bool) { } } if s.db.prefetcher != nil && prefetch && len(slotsToPrefetch) > 0 && s.data.Root != emptyRoot { - s.db.prefetcher.prefetch(s.data.Root, slotsToPrefetch) + s.db.prefetcher.prefetch(s.addrHash, s.data.Root, slotsToPrefetch) } if len(s.dirtyStorage) > 0 { s.dirtyStorage = make(Storage) @@ -352,7 +353,7 @@ func (s *stateObject) updateTrie(db Database) Trie { usedStorage = append(usedStorage, common.CopyBytes(key[:])) // Copy needed for closure } if s.db.prefetcher != nil { - s.db.prefetcher.used(s.data.Root, usedStorage) + s.db.prefetcher.used(s.addrHash, s.data.Root, usedStorage) } if len(s.pendingStorage) > 0 { s.pendingStorage = make(Storage) @@ -375,23 +376,23 @@ func (s *stateObject) updateRoot(db Database) { // CommitTrie the storage trie of the object to db. // This updates the trie root. -func (s *stateObject) CommitTrie(db Database) (int, error) { +func (s *stateObject) CommitTrie(db Database) (*trie.NodeSet, error) { // If nothing changed, don't bother with hashing anything if s.updateTrie(db) == nil { - return 0, nil + return nil, nil } if s.dbErr != nil { - return 0, s.dbErr + return nil, s.dbErr } // Track the amount of time wasted on committing the storage trie if metrics.EnabledExpensive { defer func(start time.Time) { s.db.StorageCommits += time.Since(start) }(time.Now()) } - root, committed, err := s.trie.Commit(nil) + root, nodes, err := s.trie.Commit(false) if err == nil { s.data.Root = root } - return committed, err + return nodes, err } // AddBalance adds amount to s's balance. diff --git a/core/state/state_test.go b/core/state/state_test.go index 0a55d7781fd18..b6b46e446fba9 100644 --- a/core/state/state_test.go +++ b/core/state/state_test.go @@ -25,6 +25,7 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/trie" ) type stateTest struct { @@ -40,7 +41,7 @@ func newStateTest() *stateTest { func TestDump(t *testing.T) { db := rawdb.NewMemoryDatabase() - sdb, _ := New(common.Hash{}, NewDatabaseWithConfig(db, nil), nil) + sdb, _ := New(common.Hash{}, NewDatabaseWithConfig(db, &trie.Config{Preimages: true}), nil) s := &stateTest{db: db, state: sdb} // generate a few entries diff --git a/core/state/statedb.go b/core/state/statedb.go index 1d31cf470be03..50eee8183c311 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -62,11 +62,14 @@ func (n *proofList) Delete(key []byte) error { // * Contracts // * Accounts type StateDB struct { - db Database - prefetcher *triePrefetcher - originalRoot common.Hash // The pre-state root, before any changes were made - trie Trie - hasher crypto.KeccakState + db Database + prefetcher *triePrefetcher + trie Trie + hasher crypto.KeccakState + + // originalRoot is the pre-state root, before any changes were made. + // It will be updated when the Commit is called. + originalRoot common.Hash snaps *snapshot.Tree snap snapshot.Snapshot @@ -481,7 +484,7 @@ func (s *StateDB) deleteStateObject(obj *stateObject) { } // Delete the account from the trie addr := obj.Address() - if err := s.trie.TryDelete(addr[:]); err != nil { + if err := s.trie.TryDeleteAccount(addr[:]); err != nil { s.setError(fmt.Errorf("deleteStateObject (%x) error: %v", addr[:], err)) } } @@ -534,20 +537,16 @@ func (s *StateDB) getDeletedStateObject(addr common.Address) *stateObject { // If snapshot unavailable or reading from it failed, load from the database if data == nil { start := time.Now() - enc, err := s.trie.TryGet(addr.Bytes()) + var err error + data, err = s.trie.TryGetAccount(addr.Bytes()) if metrics.EnabledExpensive { s.AccountReads += time.Since(start) } if err != nil { - s.setError(fmt.Errorf("getDeleteStateObject (%x) error: %v", addr.Bytes(), err)) - return nil - } - if len(enc) == 0 { + s.setError(fmt.Errorf("getDeleteStateObject (%x) error: %w", addr.Bytes(), err)) return nil } - data = new(types.StateAccount) - if err := rlp.DecodeBytes(enc, data); err != nil { - log.Error("Failed to decode state object", "addr", addr, "err", err) + if data == nil { return nil } } @@ -648,6 +647,7 @@ func (s *StateDB) Copy() *StateDB { state := &StateDB{ db: s.db, trie: s.db.CopyTrie(s.trie), + originalRoot: s.originalRoot, stateObjects: make(map[common.Address]*stateObject, len(s.journal.dirties)), stateObjectsPending: make(map[common.Address]struct{}, len(s.stateObjectsPending)), stateObjectsDirty: make(map[common.Address]struct{}, len(s.journal.dirties)), @@ -770,7 +770,7 @@ func (s *StateDB) GetRefund() uint64 { return s.refund } -// Finalise finalises the state by removing the s destructed objects and clears +// Finalise finalises the state by removing the destructed objects and clears // the journal as well as the refunds. Finalise, however, will not push any updates // into the tries just yet. Only IntermediateRoot or Commit will do that. func (s *StateDB) Finalise(deleteEmptyObjects bool) { @@ -792,7 +792,7 @@ func (s *StateDB) Finalise(deleteEmptyObjects bool) { // If state snapshotting is active, also mark the destruction there. // Note, we can't do this only at the end of a block because multiple // transactions within the same block might self destruct and then - // ressurrect an account; but the snapshotter needs both events. + // resurrect an account; but the snapshotter needs both events. if s.snap != nil { s.snapDestructs[obj.addrHash] = struct{}{} // We need to maintain account deletions explicitly (will remain set indefinitely) delete(s.snapAccounts, obj.addrHash) // Clear out any previously updated account data (may be recreated via a ressurrect) @@ -810,7 +810,7 @@ func (s *StateDB) Finalise(deleteEmptyObjects bool) { addressesToPrefetch = append(addressesToPrefetch, common.CopyBytes(addr[:])) // Copy needed for closure } if s.prefetcher != nil && len(addressesToPrefetch) > 0 { - s.prefetcher.prefetch(s.originalRoot, addressesToPrefetch) + s.prefetcher.prefetch(common.Hash{}, s.originalRoot, addressesToPrefetch) } // Invalidate journal because reverting across transactions is not allowed. s.clearJournalAndRefund() @@ -840,7 +840,7 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash { // Although naively it makes sense to retrieve the account trie and then do // the contract storage and account updates sequentially, that short circuits // the account prefetcher. Instead, let's process all the storage updates - // first, giving the account prefeches just a few more milliseconds of time + // first, giving the account prefetches just a few more milliseconds of time // to pull useful data from disk. for addr := range s.stateObjectsPending { if obj := s.stateObjects[addr]; !obj.deleted { @@ -851,7 +851,7 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash { // _untouched_. We can check with the prefetcher, if it can give us a trie // which has the same root, but also has some content loaded into it. if prefetcher != nil { - if trie := prefetcher.trie(s.originalRoot); trie != nil { + if trie := prefetcher.trie(common.Hash{}, s.originalRoot); trie != nil { s.trie = trie } } @@ -867,7 +867,7 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash { usedAddrs = append(usedAddrs, common.CopyBytes(addr[:])) // Copy needed for closure } if prefetcher != nil { - prefetcher.used(s.originalRoot, usedAddrs) + prefetcher.used(common.Hash{}, s.originalRoot, usedAddrs) } if len(s.stateObjectsPending) > 0 { s.stateObjectsPending = make(map[common.Address]struct{}) @@ -891,7 +891,7 @@ func (s *StateDB) clearJournalAndRefund() { s.journal = newJournal() s.refund = 0 } - s.validRevisions = s.validRevisions[:0] // Snapshots can be created without journal entires + s.validRevisions = s.validRevisions[:0] // Snapshots can be created without journal entries } // Commit writes the state to the underlying in-memory trie database. @@ -903,7 +903,11 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { s.IntermediateRoot(deleteEmptyObjects) // Commit objects to the trie, measuring the elapsed time - var storageCommitted int + var ( + accountTrieNodes int + storageTrieNodes int + nodes = trie.NewMergedNodeSet() + ) codeWriter := s.db.TrieDB().DiskDB().NewBatch() for addr := range s.stateObjectsDirty { if obj := s.stateObjects[addr]; !obj.deleted { @@ -913,11 +917,17 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { obj.dirtyCode = false } // Write any storage changes in the state object to its storage trie - committed, err := obj.CommitTrie(s.db) + set, err := obj.CommitTrie(s.db) if err != nil { return common.Hash{}, err } - storageCommitted += committed + // Merge the dirty nodes of storage trie into global set + if set != nil { + if err := nodes.Merge(set); err != nil { + return common.Hash{}, err + } + storageTrieNodes += set.Len() + } } } if len(s.stateObjectsDirty) > 0 { @@ -928,26 +938,22 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { log.Crit("Failed to commit dirty codes", "error", err) } } - // Write the account trie changes, measuing the amount of wasted time + // Write the account trie changes, measuring the amount of wasted time var start time.Time if metrics.EnabledExpensive { start = time.Now() } - // The onleaf func is called _serially_, so we can reuse the same account - // for unmarshalling every time. - var account types.StateAccount - root, accountCommitted, err := s.trie.Commit(func(_ [][]byte, _ []byte, leaf []byte, parent common.Hash) error { - if err := rlp.DecodeBytes(leaf, &account); err != nil { - return nil - } - if account.Root != emptyRoot { - s.db.TrieDB().Reference(account.Root, parent) - } - return nil - }) + root, set, err := s.trie.Commit(true) if err != nil { return common.Hash{}, err } + // Merge the dirty nodes of account trie into global set + if set != nil { + if err := nodes.Merge(set); err != nil { + return common.Hash{}, err + } + accountTrieNodes = set.Len() + } if metrics.EnabledExpensive { s.AccountCommits += time.Since(start) @@ -955,8 +961,8 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { storageUpdatedMeter.Mark(int64(s.StorageUpdated)) accountDeletedMeter.Mark(int64(s.AccountDeleted)) storageDeletedMeter.Mark(int64(s.StorageDeleted)) - accountCommittedMeter.Mark(int64(accountCommitted)) - storageCommittedMeter.Mark(int64(storageCommitted)) + accountTrieCommittedMeter.Mark(int64(accountTrieNodes)) + storageTriesCommittedMeter.Mark(int64(storageTrieNodes)) s.AccountUpdated, s.AccountDeleted = 0, 0 s.StorageUpdated, s.StorageDeleted = 0, 0 } @@ -980,6 +986,10 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { } s.snap, s.snapDestructs, s.snapAccounts, s.snapStorage = nil, nil, nil, nil } + if err := s.db.TrieDB().Update(nodes); err != nil { + return common.Hash{}, err + } + s.originalRoot = root return root, err } diff --git a/core/state/statedb_test.go b/core/state/statedb_test.go index e9576d4dc44d3..6fe36a7ecffdf 100644 --- a/core/state/statedb_test.go +++ b/core/state/statedb_test.go @@ -699,7 +699,6 @@ func TestDeleteCreateRevert(t *testing.T) { // the Commit operation fails with an error // If we are missing trie nodes, we should not continue writing to the trie func TestMissingTrieNodes(t *testing.T) { - // Create an initial state with a few accounts memDb := rawdb.NewMemoryDatabase() db := NewDatabase(memDb) @@ -772,7 +771,7 @@ func TestStateDBAccessList(t *testing.T) { t.Fatalf("expected %x to be in access list", address) } } - // Check that only the expected addresses are present in the acesslist + // Check that only the expected addresses are present in the access list for address := range state.accessList.addresses { if _, exist := addressMap[address]; !exist { t.Fatalf("extra address %x in access list", address) @@ -915,3 +914,43 @@ func TestStateDBAccessList(t *testing.T) { t.Fatalf("expected empty, got %d", got) } } + +// Tests that account and storage tries are flushed in the correct order and that +// no data loss occurs. +func TestFlushOrderDataLoss(t *testing.T) { + // Create a state trie with many accounts and slots + var ( + memdb = rawdb.NewMemoryDatabase() + statedb = NewDatabase(memdb) + state, _ = New(common.Hash{}, statedb, nil) + ) + for a := byte(0); a < 10; a++ { + state.CreateAccount(common.Address{a}) + for s := byte(0); s < 10; s++ { + state.SetState(common.Address{a}, common.Hash{a, s}, common.Hash{a, s}) + } + } + root, err := state.Commit(false) + if err != nil { + t.Fatalf("failed to commit state trie: %v", err) + } + statedb.TrieDB().Reference(root, common.Hash{}) + if err := statedb.TrieDB().Cap(1024); err != nil { + t.Fatalf("failed to cap trie dirty cache: %v", err) + } + if err := statedb.TrieDB().Commit(root, false, nil); err != nil { + t.Fatalf("failed to commit state trie: %v", err) + } + // Reopen the state trie from flushed disk and verify it + state, err = New(root, NewDatabase(memdb), nil) + if err != nil { + t.Fatalf("failed to reopen state trie: %v", err) + } + for a := byte(0); a < 10; a++ { + for s := byte(0); s < 10; s++ { + if have := state.GetState(common.Address{a}, common.Hash{a, s}); have != (common.Hash{a, s}) { + t.Errorf("account %d: slot %d: state mismatch: have %x, want %x", a, s, have, common.Hash{a, s}) + } + } + } +} diff --git a/core/state/sync.go b/core/state/sync.go index cc7d01a2188dd..00a4c67aa3cb7 100644 --- a/core/state/sync.go +++ b/core/state/sync.go @@ -27,20 +27,20 @@ import ( ) // NewStateSync create a new state trie download scheduler. -func NewStateSync(root common.Hash, database ethdb.KeyValueReader, onLeaf func(paths [][]byte, leaf []byte) error) *trie.Sync { +func NewStateSync(root common.Hash, database ethdb.KeyValueReader, onLeaf func(keys [][]byte, leaf []byte) error) *trie.Sync { // Register the storage slot callback if the external callback is specified. - var onSlot func(paths [][]byte, hexpath []byte, leaf []byte, parent common.Hash) error + var onSlot func(keys [][]byte, path []byte, leaf []byte, parent common.Hash, parentPath []byte) error if onLeaf != nil { - onSlot = func(paths [][]byte, hexpath []byte, leaf []byte, parent common.Hash) error { - return onLeaf(paths, leaf) + onSlot = func(keys [][]byte, path []byte, leaf []byte, parent common.Hash, parentPath []byte) error { + return onLeaf(keys, leaf) } } // Register the account callback to connect the state trie and the storage // trie belongs to the contract. var syncer *trie.Sync - onAccount := func(paths [][]byte, hexpath []byte, leaf []byte, parent common.Hash) error { + onAccount := func(keys [][]byte, path []byte, leaf []byte, parent common.Hash, parentPath []byte) error { if onLeaf != nil { - if err := onLeaf(paths, leaf); err != nil { + if err := onLeaf(keys, leaf); err != nil { return err } } @@ -48,8 +48,8 @@ func NewStateSync(root common.Hash, database ethdb.KeyValueReader, onLeaf func(p if err := rlp.Decode(bytes.NewReader(leaf), &obj); err != nil { return err } - syncer.AddSubTrie(obj.Root, hexpath, parent, onSlot) - syncer.AddCodeEntry(common.BytesToHash(obj.CodeHash), hexpath, parent) + syncer.AddSubTrie(obj.Root, path, parent, parentPath, onSlot) + syncer.AddCodeEntry(common.BytesToHash(obj.CodeHash), path, parent, parentPath) return nil } syncer = trie.NewSync(root, database, onAccount) diff --git a/core/state/sync_test.go b/core/state/sync_test.go index 007590c76d9c7..3d9fe556d2ad4 100644 --- a/core/state/sync_test.go +++ b/core/state/sync_test.go @@ -104,7 +104,7 @@ func checkTrieConsistency(db ethdb.Database, root common.Hash) error { if v, _ := db.Get(root[:]); v == nil { return nil // Consider a non existent state consistent. } - trie, err := trie.New(root, trie.NewDatabase(db)) + trie, err := trie.New(common.Hash{}, root, trie.NewDatabase(db)) if err != nil { return err } @@ -134,8 +134,8 @@ func checkStateConsistency(db ethdb.Database, root common.Hash) error { func TestEmptyStateSync(t *testing.T) { empty := common.HexToHash("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421") sync := NewStateSync(empty, rawdb.NewMemoryDatabase(), nil) - if nodes, paths, codes := sync.Missing(1); len(nodes) != 0 || len(paths) != 0 || len(codes) != 0 { - t.Errorf(" content requested for empty state: %v, %v, %v", nodes, paths, codes) + if paths, nodes, codes := sync.Missing(1); len(paths) != 0 || len(nodes) != 0 || len(codes) != 0 { + t.Errorf("content requested for empty state: %v, %v, %v", nodes, paths, codes) } } @@ -160,66 +160,93 @@ func TestIterativeStateSyncBatchedByPath(t *testing.T) { testIterativeStateSync(t, 100, false, true) } +// stateElement represents the element in the state trie(bytecode or trie node). +type stateElement struct { + path string + hash common.Hash + code common.Hash + syncPath trie.SyncPath +} + func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool) { // Create a random state to copy srcDb, srcRoot, srcAccounts := makeTestState() if commit { srcDb.TrieDB().Commit(srcRoot, false, nil) } - srcTrie, _ := trie.New(srcRoot, srcDb.TrieDB()) + srcTrie, _ := trie.New(common.Hash{}, srcRoot, srcDb.TrieDB()) // Create a destination state and sync with the scheduler dstDb := rawdb.NewMemoryDatabase() sched := NewStateSync(srcRoot, dstDb, nil) - nodes, paths, codes := sched.Missing(count) var ( - hashQueue []common.Hash - pathQueue []trie.SyncPath + nodeElements []stateElement + codeElements []stateElement ) - if !bypath { - hashQueue = append(append(hashQueue[:0], nodes...), codes...) - } else { - hashQueue = append(hashQueue[:0], codes...) - pathQueue = append(pathQueue[:0], paths...) + paths, nodes, codes := sched.Missing(count) + for i := 0; i < len(paths); i++ { + nodeElements = append(nodeElements, stateElement{ + path: paths[i], + hash: nodes[i], + syncPath: trie.NewSyncPath([]byte(paths[i])), + }) } - for len(hashQueue)+len(pathQueue) > 0 { - results := make([]trie.SyncResult, len(hashQueue)+len(pathQueue)) - for i, hash := range hashQueue { - data, err := srcDb.TrieDB().Node(hash) - if err != nil { - data, err = srcDb.ContractCode(common.Hash{}, hash) - } + for i := 0; i < len(codes); i++ { + codeElements = append(codeElements, stateElement{ + code: codes[i], + }) + } + for len(nodeElements)+len(codeElements) > 0 { + var ( + nodeResults = make([]trie.NodeSyncResult, len(nodeElements)) + codeResults = make([]trie.CodeSyncResult, len(codeElements)) + ) + for i, element := range codeElements { + data, err := srcDb.ContractCode(common.Hash{}, element.code) if err != nil { - t.Fatalf("failed to retrieve node data for hash %x", hash) + t.Fatalf("failed to retrieve contract bytecode for hash %x", element.code) } - results[i] = trie.SyncResult{Hash: hash, Data: data} + codeResults[i] = trie.CodeSyncResult{Hash: element.code, Data: data} } - for i, path := range pathQueue { - if len(path) == 1 { - data, _, err := srcTrie.TryGetNode(path[0]) - if err != nil { - t.Fatalf("failed to retrieve node data for path %x: %v", path, err) + for i, node := range nodeElements { + if bypath { + if len(node.syncPath) == 1 { + data, _, err := srcTrie.TryGetNode(node.syncPath[0]) + if err != nil { + t.Fatalf("failed to retrieve node data for path %x: %v", node.syncPath[0], err) + } + nodeResults[i] = trie.NodeSyncResult{Path: node.path, Data: data} + } else { + var acc types.StateAccount + if err := rlp.DecodeBytes(srcTrie.Get(node.syncPath[0]), &acc); err != nil { + t.Fatalf("failed to decode account on path %x: %v", node.syncPath[0], err) + } + stTrie, err := trie.New(common.BytesToHash(node.syncPath[0]), acc.Root, srcDb.TrieDB()) + if err != nil { + t.Fatalf("failed to retriev storage trie for path %x: %v", node.syncPath[1], err) + } + data, _, err := stTrie.TryGetNode(node.syncPath[1]) + if err != nil { + t.Fatalf("failed to retrieve node data for path %x: %v", node.syncPath[1], err) + } + nodeResults[i] = trie.NodeSyncResult{Path: node.path, Data: data} } - results[len(hashQueue)+i] = trie.SyncResult{Hash: crypto.Keccak256Hash(data), Data: data} } else { - var acc types.StateAccount - if err := rlp.DecodeBytes(srcTrie.Get(path[0]), &acc); err != nil { - t.Fatalf("failed to decode account on path %x: %v", path, err) - } - stTrie, err := trie.New(acc.Root, srcDb.TrieDB()) + data, err := srcDb.TrieDB().Node(node.hash) if err != nil { - t.Fatalf("failed to retriev storage trie for path %x: %v", path, err) + t.Fatalf("failed to retrieve node data for key %v", []byte(node.path)) } - data, _, err := stTrie.TryGetNode(path[1]) - if err != nil { - t.Fatalf("failed to retrieve node data for path %x: %v", path, err) - } - results[len(hashQueue)+i] = trie.SyncResult{Hash: crypto.Keccak256Hash(data), Data: data} + nodeResults[i] = trie.NodeSyncResult{Path: node.path, Data: data} } } - for _, result := range results { - if err := sched.Process(result); err != nil { + for _, result := range codeResults { + if err := sched.ProcessCode(result); err != nil { + t.Errorf("failed to process result %v", err) + } + } + for _, result := range nodeResults { + if err := sched.ProcessNode(result); err != nil { t.Errorf("failed to process result %v", err) } } @@ -229,12 +256,20 @@ func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool) { } batch.Write() - nodes, paths, codes = sched.Missing(count) - if !bypath { - hashQueue = append(append(hashQueue[:0], nodes...), codes...) - } else { - hashQueue = append(hashQueue[:0], codes...) - pathQueue = append(pathQueue[:0], paths...) + paths, nodes, codes = sched.Missing(count) + nodeElements = nodeElements[:0] + for i := 0; i < len(paths); i++ { + nodeElements = append(nodeElements, stateElement{ + path: paths[i], + hash: nodes[i], + syncPath: trie.NewSyncPath([]byte(paths[i])), + }) + } + codeElements = codeElements[:0] + for i := 0; i < len(codes); i++ { + codeElements = append(codeElements, stateElement{ + code: codes[i], + }) } } // Cross check that the two states are in sync @@ -251,26 +286,58 @@ func TestIterativeDelayedStateSync(t *testing.T) { dstDb := rawdb.NewMemoryDatabase() sched := NewStateSync(srcRoot, dstDb, nil) - nodes, _, codes := sched.Missing(0) - queue := append(append([]common.Hash{}, nodes...), codes...) - - for len(queue) > 0 { + var ( + nodeElements []stateElement + codeElements []stateElement + ) + paths, nodes, codes := sched.Missing(0) + for i := 0; i < len(paths); i++ { + nodeElements = append(nodeElements, stateElement{ + path: paths[i], + hash: nodes[i], + syncPath: trie.NewSyncPath([]byte(paths[i])), + }) + } + for i := 0; i < len(codes); i++ { + codeElements = append(codeElements, stateElement{ + code: codes[i], + }) + } + for len(nodeElements)+len(codeElements) > 0 { // Sync only half of the scheduled nodes - results := make([]trie.SyncResult, len(queue)/2+1) - for i, hash := range queue[:len(results)] { - data, err := srcDb.TrieDB().Node(hash) - if err != nil { - data, err = srcDb.ContractCode(common.Hash{}, hash) + var nodeProcessed int + var codeProcessed int + if len(codeElements) > 0 { + codeResults := make([]trie.CodeSyncResult, len(codeElements)/2+1) + for i, element := range codeElements[:len(codeResults)] { + data, err := srcDb.ContractCode(common.Hash{}, element.code) + if err != nil { + t.Fatalf("failed to retrieve contract bytecode for %x", element.code) + } + codeResults[i] = trie.CodeSyncResult{Hash: element.code, Data: data} } - if err != nil { - t.Fatalf("failed to retrieve node data for %x", hash) + for _, result := range codeResults { + if err := sched.ProcessCode(result); err != nil { + t.Fatalf("failed to process result %v", err) + } } - results[i] = trie.SyncResult{Hash: hash, Data: data} + codeProcessed = len(codeResults) } - for _, result := range results { - if err := sched.Process(result); err != nil { - t.Fatalf("failed to process result %v", err) + if len(nodeElements) > 0 { + nodeResults := make([]trie.NodeSyncResult, len(nodeElements)/2+1) + for i, element := range nodeElements[:len(nodeResults)] { + data, err := srcDb.TrieDB().Node(element.hash) + if err != nil { + t.Fatalf("failed to retrieve contract bytecode for %x", element.code) + } + nodeResults[i] = trie.NodeSyncResult{Path: element.path, Data: data} + } + for _, result := range nodeResults { + if err := sched.ProcessNode(result); err != nil { + t.Fatalf("failed to process result %v", err) + } } + nodeProcessed = len(nodeResults) } batch := dstDb.NewBatch() if err := sched.Commit(batch); err != nil { @@ -278,8 +345,21 @@ func TestIterativeDelayedStateSync(t *testing.T) { } batch.Write() - nodes, _, codes = sched.Missing(0) - queue = append(append(queue[len(results):], nodes...), codes...) + paths, nodes, codes = sched.Missing(0) + nodeElements = nodeElements[nodeProcessed:] + for i := 0; i < len(paths); i++ { + nodeElements = append(nodeElements, stateElement{ + path: paths[i], + hash: nodes[i], + syncPath: trie.NewSyncPath([]byte(paths[i])), + }) + } + codeElements = codeElements[codeProcessed:] + for i := 0; i < len(codes); i++ { + codeElements = append(codeElements, stateElement{ + code: codes[i], + }) + } } // Cross check that the two states are in sync checkStateAccounts(t, dstDb, srcRoot, srcAccounts) @@ -299,40 +379,70 @@ func testIterativeRandomStateSync(t *testing.T, count int) { dstDb := rawdb.NewMemoryDatabase() sched := NewStateSync(srcRoot, dstDb, nil) - queue := make(map[common.Hash]struct{}) - nodes, _, codes := sched.Missing(count) - for _, hash := range append(nodes, codes...) { - queue[hash] = struct{}{} + nodeQueue := make(map[string]stateElement) + codeQueue := make(map[common.Hash]struct{}) + paths, nodes, codes := sched.Missing(count) + for i, path := range paths { + nodeQueue[path] = stateElement{ + path: path, + hash: nodes[i], + syncPath: trie.NewSyncPath([]byte(path)), + } } - for len(queue) > 0 { + for _, hash := range codes { + codeQueue[hash] = struct{}{} + } + for len(nodeQueue)+len(codeQueue) > 0 { // Fetch all the queued nodes in a random order - results := make([]trie.SyncResult, 0, len(queue)) - for hash := range queue { - data, err := srcDb.TrieDB().Node(hash) - if err != nil { - data, err = srcDb.ContractCode(common.Hash{}, hash) + if len(codeQueue) > 0 { + results := make([]trie.CodeSyncResult, 0, len(codeQueue)) + for hash := range codeQueue { + data, err := srcDb.ContractCode(common.Hash{}, hash) + if err != nil { + t.Fatalf("failed to retrieve node data for %x", hash) + } + results = append(results, trie.CodeSyncResult{Hash: hash, Data: data}) } - if err != nil { - t.Fatalf("failed to retrieve node data for %x", hash) + for _, result := range results { + if err := sched.ProcessCode(result); err != nil { + t.Fatalf("failed to process result %v", err) + } } - results = append(results, trie.SyncResult{Hash: hash, Data: data}) } - // Feed the retrieved results back and queue new tasks - for _, result := range results { - if err := sched.Process(result); err != nil { - t.Fatalf("failed to process result %v", err) + if len(nodeQueue) > 0 { + results := make([]trie.NodeSyncResult, 0, len(nodeQueue)) + for path, element := range nodeQueue { + data, err := srcDb.TrieDB().Node(element.hash) + if err != nil { + t.Fatalf("failed to retrieve node data for %x %v %v", element.hash, []byte(element.path), element.path) + } + results = append(results, trie.NodeSyncResult{Path: path, Data: data}) + } + for _, result := range results { + if err := sched.ProcessNode(result); err != nil { + t.Fatalf("failed to process result %v", err) + } } } + // Feed the retrieved results back and queue new tasks batch := dstDb.NewBatch() if err := sched.Commit(batch); err != nil { t.Fatalf("failed to commit data: %v", err) } batch.Write() - queue = make(map[common.Hash]struct{}) - nodes, _, codes = sched.Missing(count) - for _, hash := range append(nodes, codes...) { - queue[hash] = struct{}{} + nodeQueue = make(map[string]stateElement) + codeQueue = make(map[common.Hash]struct{}) + paths, nodes, codes := sched.Missing(count) + for i, path := range paths { + nodeQueue[path] = stateElement{ + path: path, + hash: nodes[i], + syncPath: trie.NewSyncPath([]byte(path)), + } + } + for _, hash := range codes { + codeQueue[hash] = struct{}{} } } // Cross check that the two states are in sync @@ -349,34 +459,62 @@ func TestIterativeRandomDelayedStateSync(t *testing.T) { dstDb := rawdb.NewMemoryDatabase() sched := NewStateSync(srcRoot, dstDb, nil) - queue := make(map[common.Hash]struct{}) - nodes, _, codes := sched.Missing(0) - for _, hash := range append(nodes, codes...) { - queue[hash] = struct{}{} + nodeQueue := make(map[string]stateElement) + codeQueue := make(map[common.Hash]struct{}) + paths, nodes, codes := sched.Missing(0) + for i, path := range paths { + nodeQueue[path] = stateElement{ + path: path, + hash: nodes[i], + syncPath: trie.NewSyncPath([]byte(path)), + } + } + for _, hash := range codes { + codeQueue[hash] = struct{}{} } - for len(queue) > 0 { + for len(nodeQueue)+len(codeQueue) > 0 { // Sync only half of the scheduled nodes, even those in random order - results := make([]trie.SyncResult, 0, len(queue)/2+1) - for hash := range queue { - delete(queue, hash) + if len(codeQueue) > 0 { + results := make([]trie.CodeSyncResult, 0, len(codeQueue)/2+1) + for hash := range codeQueue { + delete(codeQueue, hash) - data, err := srcDb.TrieDB().Node(hash) - if err != nil { - data, err = srcDb.ContractCode(common.Hash{}, hash) + data, err := srcDb.ContractCode(common.Hash{}, hash) + if err != nil { + t.Fatalf("failed to retrieve node data for %x", hash) + } + results = append(results, trie.CodeSyncResult{Hash: hash, Data: data}) + + if len(results) >= cap(results) { + break + } } - if err != nil { - t.Fatalf("failed to retrieve node data for %x", hash) + for _, result := range results { + if err := sched.ProcessCode(result); err != nil { + t.Fatalf("failed to process result %v", err) + } } - results = append(results, trie.SyncResult{Hash: hash, Data: data}) + } + if len(nodeQueue) > 0 { + results := make([]trie.NodeSyncResult, 0, len(nodeQueue)/2+1) + for path, element := range nodeQueue { + delete(nodeQueue, path) - if len(results) >= cap(results) { - break + data, err := srcDb.TrieDB().Node(element.hash) + if err != nil { + t.Fatalf("failed to retrieve node data for %x", element.hash) + } + results = append(results, trie.NodeSyncResult{Path: path, Data: data}) + + if len(results) >= cap(results) { + break + } } - } - // Feed the retrieved results back and queue new tasks - for _, result := range results { - if err := sched.Process(result); err != nil { - t.Fatalf("failed to process result %v", err) + // Feed the retrieved results back and queue new tasks + for _, result := range results { + if err := sched.ProcessNode(result); err != nil { + t.Fatalf("failed to process result %v", err) + } } } batch := dstDb.NewBatch() @@ -384,12 +522,17 @@ func TestIterativeRandomDelayedStateSync(t *testing.T) { t.Fatalf("failed to commit data: %v", err) } batch.Write() - for _, result := range results { - delete(queue, result.Hash) + + paths, nodes, codes := sched.Missing(0) + for i, path := range paths { + nodeQueue[path] = stateElement{ + path: path, + hash: nodes[i], + syncPath: trie.NewSyncPath([]byte(path)), + } } - nodes, _, codes = sched.Missing(0) - for _, hash := range append(nodes, codes...) { - queue[hash] = struct{}{} + for _, hash := range codes { + codeQueue[hash] = struct{}{} } } // Cross check that the two states are in sync @@ -416,28 +559,62 @@ func TestIncompleteStateSync(t *testing.T) { dstDb := rawdb.NewMemoryDatabase() sched := NewStateSync(srcRoot, dstDb, nil) - var added []common.Hash - - nodes, _, codes := sched.Missing(1) - queue := append(append([]common.Hash{}, nodes...), codes...) - - for len(queue) > 0 { + var ( + addedCodes []common.Hash + addedNodes []common.Hash + ) + nodeQueue := make(map[string]stateElement) + codeQueue := make(map[common.Hash]struct{}) + paths, nodes, codes := sched.Missing(1) + for i, path := range paths { + nodeQueue[path] = stateElement{ + path: path, + hash: nodes[i], + syncPath: trie.NewSyncPath([]byte(path)), + } + } + for _, hash := range codes { + codeQueue[hash] = struct{}{} + } + for len(nodeQueue)+len(codeQueue) > 0 { // Fetch a batch of state nodes - results := make([]trie.SyncResult, len(queue)) - for i, hash := range queue { - data, err := srcDb.TrieDB().Node(hash) - if err != nil { - data, err = srcDb.ContractCode(common.Hash{}, hash) + if len(codeQueue) > 0 { + results := make([]trie.CodeSyncResult, 0, len(codeQueue)) + for hash := range codeQueue { + data, err := srcDb.ContractCode(common.Hash{}, hash) + if err != nil { + t.Fatalf("failed to retrieve node data for %x", hash) + } + results = append(results, trie.CodeSyncResult{Hash: hash, Data: data}) + addedCodes = append(addedCodes, hash) } - if err != nil { - t.Fatalf("failed to retrieve node data for %x", hash) + // Process each of the state nodes + for _, result := range results { + if err := sched.ProcessCode(result); err != nil { + t.Fatalf("failed to process result %v", err) + } } - results[i] = trie.SyncResult{Hash: hash, Data: data} } - // Process each of the state nodes - for _, result := range results { - if err := sched.Process(result); err != nil { - t.Fatalf("failed to process result %v", err) + var nodehashes []common.Hash + if len(nodeQueue) > 0 { + results := make([]trie.NodeSyncResult, 0, len(nodeQueue)) + for key, element := range nodeQueue { + data, err := srcDb.TrieDB().Node(element.hash) + if err != nil { + t.Fatalf("failed to retrieve node data for %x", element.hash) + } + results = append(results, trie.NodeSyncResult{Path: key, Data: data}) + + if element.hash != srcRoot { + addedNodes = append(addedNodes, element.hash) + } + nodehashes = append(nodehashes, element.hash) + } + // Process each of the state nodes + for _, result := range results { + if err := sched.ProcessNode(result); err != nil { + t.Fatalf("failed to process result %v", err) + } } } batch := dstDb.NewBatch() @@ -445,43 +622,44 @@ func TestIncompleteStateSync(t *testing.T) { t.Fatalf("failed to commit data: %v", err) } batch.Write() - for _, result := range results { - added = append(added, result.Hash) - // Check that all known sub-tries added so far are complete or missing entirely. - if _, ok := isCode[result.Hash]; ok { - continue - } + + for _, root := range nodehashes { // Can't use checkStateConsistency here because subtrie keys may have odd // length and crash in LeafKey. - if err := checkTrieConsistency(dstDb, result.Hash); err != nil { + if err := checkTrieConsistency(dstDb, root); err != nil { t.Fatalf("state inconsistent: %v", err) } } // Fetch the next batch to retrieve - nodes, _, codes = sched.Missing(1) - queue = append(append(queue[:0], nodes...), codes...) + nodeQueue = make(map[string]stateElement) + codeQueue = make(map[common.Hash]struct{}) + paths, nodes, codes := sched.Missing(1) + for i, path := range paths { + nodeQueue[path] = stateElement{ + path: path, + hash: nodes[i], + syncPath: trie.NewSyncPath([]byte(path)), + } + } + for _, hash := range codes { + codeQueue[hash] = struct{}{} + } } // Sanity check that removing any node from the database is detected - for _, node := range added[1:] { - var ( - key = node.Bytes() - _, code = isCode[node] - val []byte - ) - if code { - val = rawdb.ReadCode(dstDb, node) - rawdb.DeleteCode(dstDb, node) - } else { - val = rawdb.ReadTrieNode(dstDb, node) - rawdb.DeleteTrieNode(dstDb, node) + for _, node := range addedCodes { + val := rawdb.ReadCode(dstDb, node) + rawdb.DeleteCode(dstDb, node) + if err := checkStateConsistency(dstDb, srcRoot); err == nil { + t.Errorf("trie inconsistency not caught, missing: %x", node) } - if err := checkStateConsistency(dstDb, added[0]); err == nil { - t.Fatalf("trie inconsistency not caught, missing: %x", key) - } - if code { - rawdb.WriteCode(dstDb, node, val) - } else { - rawdb.WriteTrieNode(dstDb, node, val) + rawdb.WriteCode(dstDb, node, val) + } + for _, node := range addedNodes { + val := rawdb.ReadTrieNode(dstDb, node) + rawdb.DeleteTrieNode(dstDb, node) + if err := checkStateConsistency(dstDb, srcRoot); err == nil { + t.Errorf("trie inconsistency not caught, missing: %v", node.Hex()) } + rawdb.WriteTrieNode(dstDb, node, val) } } diff --git a/core/state/trie_prefetcher.go b/core/state/trie_prefetcher.go index 25c3730e3f7af..83e8966d4c9f5 100644 --- a/core/state/trie_prefetcher.go +++ b/core/state/trie_prefetcher.go @@ -25,7 +25,7 @@ import ( ) var ( - // triePrefetchMetricsPrefix is the prefix under which to publis the metrics. + // triePrefetchMetricsPrefix is the prefix under which to publish the metrics. triePrefetchMetricsPrefix = "trie/prefetch/" ) @@ -35,10 +35,10 @@ var ( // // Note, the prefetcher's API is not thread safe. type triePrefetcher struct { - db Database // Database to fetch trie nodes through - root common.Hash // Root hash of theaccount trie for metrics - fetches map[common.Hash]Trie // Partially or fully fetcher tries - fetchers map[common.Hash]*subfetcher // Subfetchers for each trie + db Database // Database to fetch trie nodes through + root common.Hash // Root hash of the account trie for metrics + fetches map[string]Trie // Partially or fully fetcher tries + fetchers map[string]*subfetcher // Subfetchers for each trie deliveryMissMeter metrics.Meter accountLoadMeter metrics.Meter @@ -51,13 +51,12 @@ type triePrefetcher struct { storageWasteMeter metrics.Meter } -// newTriePrefetcher func newTriePrefetcher(db Database, root common.Hash, namespace string) *triePrefetcher { prefix := triePrefetchMetricsPrefix + namespace p := &triePrefetcher{ db: db, root: root, - fetchers: make(map[common.Hash]*subfetcher), // Active prefetchers use the fetchers map + fetchers: make(map[string]*subfetcher), // Active prefetchers use the fetchers map deliveryMissMeter: metrics.GetOrRegisterMeter(prefix+"/deliverymiss", nil), accountLoadMeter: metrics.GetOrRegisterMeter(prefix+"/account/load", nil), @@ -112,7 +111,7 @@ func (p *triePrefetcher) copy() *triePrefetcher { copy := &triePrefetcher{ db: p.db, root: p.root, - fetches: make(map[common.Hash]Trie), // Active prefetchers use the fetches map + fetches: make(map[string]Trie), // Active prefetchers use the fetches map deliveryMissMeter: p.deliveryMissMeter, accountLoadMeter: p.accountLoadMeter, @@ -132,33 +131,35 @@ func (p *triePrefetcher) copy() *triePrefetcher { return copy } // Otherwise we're copying an active fetcher, retrieve the current states - for root, fetcher := range p.fetchers { - copy.fetches[root] = fetcher.peek() + for id, fetcher := range p.fetchers { + copy.fetches[id] = fetcher.peek() } return copy } // prefetch schedules a batch of trie items to prefetch. -func (p *triePrefetcher) prefetch(root common.Hash, keys [][]byte) { +func (p *triePrefetcher) prefetch(owner common.Hash, root common.Hash, keys [][]byte) { // If the prefetcher is an inactive one, bail out if p.fetches != nil { return } // Active fetcher, schedule the retrievals - fetcher := p.fetchers[root] + id := p.trieID(owner, root) + fetcher := p.fetchers[id] if fetcher == nil { - fetcher = newSubfetcher(p.db, root) - p.fetchers[root] = fetcher + fetcher = newSubfetcher(p.db, owner, root) + p.fetchers[id] = fetcher } fetcher.schedule(keys) } // trie returns the trie matching the root hash, or nil if the prefetcher doesn't // have it. -func (p *triePrefetcher) trie(root common.Hash) Trie { +func (p *triePrefetcher) trie(owner common.Hash, root common.Hash) Trie { // If the prefetcher is inactive, return from existing deep copies + id := p.trieID(owner, root) if p.fetches != nil { - trie := p.fetches[root] + trie := p.fetches[id] if trie == nil { p.deliveryMissMeter.Mark(1) return nil @@ -166,7 +167,7 @@ func (p *triePrefetcher) trie(root common.Hash) Trie { return p.db.CopyTrie(trie) } // Otherwise the prefetcher is active, bail if no trie was prefetched for this root - fetcher := p.fetchers[root] + fetcher := p.fetchers[id] if fetcher == nil { p.deliveryMissMeter.Mark(1) return nil @@ -185,27 +186,33 @@ func (p *triePrefetcher) trie(root common.Hash) Trie { // used marks a batch of state items used to allow creating statistics as to // how useful or wasteful the prefetcher is. -func (p *triePrefetcher) used(root common.Hash, used [][]byte) { - if fetcher := p.fetchers[root]; fetcher != nil { +func (p *triePrefetcher) used(owner common.Hash, root common.Hash, used [][]byte) { + if fetcher := p.fetchers[p.trieID(owner, root)]; fetcher != nil { fetcher.used = used } } +// trieID returns an unique trie identifier consists the trie owner and root hash. +func (p *triePrefetcher) trieID(owner common.Hash, root common.Hash) string { + return string(append(owner.Bytes(), root.Bytes()...)) +} + // subfetcher is a trie fetcher goroutine responsible for pulling entries for a // single trie. It is spawned when a new root is encountered and lives until the // main prefetcher is paused and either all requested items are processed or if // the trie being worked on is retrieved from the prefetcher. type subfetcher struct { - db Database // Database to load trie nodes through - root common.Hash // Root hash of the trie to prefetch - trie Trie // Trie being populated with nodes + db Database // Database to load trie nodes through + owner common.Hash // Owner of the trie, usually account hash + root common.Hash // Root hash of the trie to prefetch + trie Trie // Trie being populated with nodes tasks [][]byte // Items queued up for retrieval lock sync.Mutex // Lock protecting the task queue wake chan struct{} // Wake channel if a new task is scheduled stop chan struct{} // Channel to interrupt processing - term chan struct{} // Channel to signal iterruption + term chan struct{} // Channel to signal interruption copy chan chan Trie // Channel to request a copy of the current trie seen map[string]struct{} // Tracks the entries already loaded @@ -215,15 +222,16 @@ type subfetcher struct { // newSubfetcher creates a goroutine to prefetch state items belonging to a // particular root hash. -func newSubfetcher(db Database, root common.Hash) *subfetcher { +func newSubfetcher(db Database, owner common.Hash, root common.Hash) *subfetcher { sf := &subfetcher{ - db: db, - root: root, - wake: make(chan struct{}, 1), - stop: make(chan struct{}), - term: make(chan struct{}), - copy: make(chan chan Trie), - seen: make(map[string]struct{}), + db: db, + owner: owner, + root: root, + wake: make(chan struct{}, 1), + stop: make(chan struct{}), + term: make(chan struct{}), + copy: make(chan chan Trie), + seen: make(map[string]struct{}), } go sf.loop() return sf @@ -279,13 +287,21 @@ func (sf *subfetcher) loop() { defer close(sf.term) // Start by opening the trie and stop processing if it fails - trie, err := sf.db.OpenTrie(sf.root) - if err != nil { - log.Warn("Trie prefetcher failed opening trie", "root", sf.root, "err", err) - return + if sf.owner == (common.Hash{}) { + trie, err := sf.db.OpenTrie(sf.root) + if err != nil { + log.Warn("Trie prefetcher failed opening trie", "root", sf.root, "err", err) + return + } + sf.trie = trie + } else { + trie, err := sf.db.OpenStorageTrie(sf.owner, sf.root) + if err != nil { + log.Warn("Trie prefetcher failed opening trie", "root", sf.root, "err", err) + return + } + sf.trie = trie } - sf.trie = trie - // Trie opened successfully, keep prefetching items for { select { @@ -315,7 +331,11 @@ func (sf *subfetcher) loop() { if _, ok := sf.seen[string(task)]; ok { sf.dups++ } else { - sf.trie.TryGet(task) + if len(task) == len(common.Address{}) { + sf.trie.TryGetAccount(task) + } else { + sf.trie.TryGet(task) + } sf.seen[string(task)] = struct{}{} } } diff --git a/core/state/trie_prefetcher_test.go b/core/state/trie_prefetcher_test.go index 35dc7a2c0da47..cb0b67d7ea79f 100644 --- a/core/state/trie_prefetcher_test.go +++ b/core/state/trie_prefetcher_test.go @@ -47,20 +47,20 @@ func TestCopyAndClose(t *testing.T) { db := filledStateDB() prefetcher := newTriePrefetcher(db.db, db.originalRoot, "") skey := common.HexToHash("aaa") - prefetcher.prefetch(db.originalRoot, [][]byte{skey.Bytes()}) - prefetcher.prefetch(db.originalRoot, [][]byte{skey.Bytes()}) + prefetcher.prefetch(common.Hash{}, db.originalRoot, [][]byte{skey.Bytes()}) + prefetcher.prefetch(common.Hash{}, db.originalRoot, [][]byte{skey.Bytes()}) time.Sleep(1 * time.Second) - a := prefetcher.trie(db.originalRoot) - prefetcher.prefetch(db.originalRoot, [][]byte{skey.Bytes()}) - b := prefetcher.trie(db.originalRoot) + a := prefetcher.trie(common.Hash{}, db.originalRoot) + prefetcher.prefetch(common.Hash{}, db.originalRoot, [][]byte{skey.Bytes()}) + b := prefetcher.trie(common.Hash{}, db.originalRoot) cpy := prefetcher.copy() - cpy.prefetch(db.originalRoot, [][]byte{skey.Bytes()}) - cpy.prefetch(db.originalRoot, [][]byte{skey.Bytes()}) - c := cpy.trie(db.originalRoot) + cpy.prefetch(common.Hash{}, db.originalRoot, [][]byte{skey.Bytes()}) + cpy.prefetch(common.Hash{}, db.originalRoot, [][]byte{skey.Bytes()}) + c := cpy.trie(common.Hash{}, db.originalRoot) prefetcher.close() cpy2 := cpy.copy() - cpy2.prefetch(db.originalRoot, [][]byte{skey.Bytes()}) - d := cpy2.trie(db.originalRoot) + cpy2.prefetch(common.Hash{}, db.originalRoot, [][]byte{skey.Bytes()}) + d := cpy2.trie(common.Hash{}, db.originalRoot) cpy.close() cpy2.close() if a.Hash() != b.Hash() || a.Hash() != c.Hash() || a.Hash() != d.Hash() { @@ -72,10 +72,10 @@ func TestUseAfterClose(t *testing.T) { db := filledStateDB() prefetcher := newTriePrefetcher(db.db, db.originalRoot, "") skey := common.HexToHash("aaa") - prefetcher.prefetch(db.originalRoot, [][]byte{skey.Bytes()}) - a := prefetcher.trie(db.originalRoot) + prefetcher.prefetch(common.Hash{}, db.originalRoot, [][]byte{skey.Bytes()}) + a := prefetcher.trie(common.Hash{}, db.originalRoot) prefetcher.close() - b := prefetcher.trie(db.originalRoot) + b := prefetcher.trie(common.Hash{}, db.originalRoot) if a == nil { t.Fatal("Prefetching before close should not return nil") } @@ -88,13 +88,13 @@ func TestCopyClose(t *testing.T) { db := filledStateDB() prefetcher := newTriePrefetcher(db.db, db.originalRoot, "") skey := common.HexToHash("aaa") - prefetcher.prefetch(db.originalRoot, [][]byte{skey.Bytes()}) + prefetcher.prefetch(common.Hash{}, db.originalRoot, [][]byte{skey.Bytes()}) cpy := prefetcher.copy() - a := prefetcher.trie(db.originalRoot) - b := cpy.trie(db.originalRoot) + a := prefetcher.trie(common.Hash{}, db.originalRoot) + b := cpy.trie(common.Hash{}, db.originalRoot) prefetcher.close() - c := prefetcher.trie(db.originalRoot) - d := cpy.trie(db.originalRoot) + c := prefetcher.trie(common.Hash{}, db.originalRoot) + d := cpy.trie(common.Hash{}, db.originalRoot) if a == nil { t.Fatal("Prefetching before close should not return nil") } diff --git a/core/state_processor.go b/core/state_processor.go index d4c77ae410421..e511697c5f6af 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -79,7 +79,7 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg return nil, nil, 0, fmt.Errorf("could not apply tx %d [%v]: %w", i, tx.Hash().Hex(), err) } statedb.Prepare(tx.Hash(), i) - receipt, err := applyTransaction(msg, p.config, p.bc, nil, gp, statedb, blockNumber, blockHash, tx, usedGas, vmenv) + receipt, err := applyTransaction(msg, p.config, nil, gp, statedb, blockNumber, blockHash, tx, usedGas, vmenv) if err != nil { return nil, nil, 0, fmt.Errorf("could not apply tx %d [%v]: %w", i, tx.Hash().Hex(), err) } @@ -92,7 +92,7 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg return receipts, allLogs, *usedGas, nil } -func applyTransaction(msg types.Message, config *params.ChainConfig, bc ChainContext, author *common.Address, gp *GasPool, statedb *state.StateDB, blockNumber *big.Int, blockHash common.Hash, tx *types.Transaction, usedGas *uint64, evm *vm.EVM) (*types.Receipt, error) { +func applyTransaction(msg types.Message, config *params.ChainConfig, author *common.Address, gp *GasPool, statedb *state.StateDB, blockNumber *big.Int, blockHash common.Hash, tx *types.Transaction, usedGas *uint64, evm *vm.EVM) (*types.Receipt, error) { // Create a new context to be used in the EVM environment. txContext := NewEVMTxContext(msg) evm.Reset(txContext, statedb) @@ -149,5 +149,5 @@ func ApplyTransaction(config *params.ChainConfig, bc ChainContext, author *commo // Create a new context to be used in the EVM environment blockContext := NewEVMBlockContext(header, bc, author) vmenv := vm.NewEVM(blockContext, vm.TxContext{}, statedb, config, cfg) - return applyTransaction(msg, config, bc, author, gp, statedb, header.Number, header.Hash(), tx, usedGas, vmenv) + return applyTransaction(msg, config, author, gp, statedb, header.Number, header.Hash(), tx, usedGas, vmenv) } diff --git a/core/state_transition.go b/core/state_transition.go index 3b5f81b166329..0946c0372e2f6 100644 --- a/core/state_transition.go +++ b/core/state_transition.go @@ -344,7 +344,16 @@ func (st *StateTransition) TransitionDb() (*ExecutionResult, error) { if rules.IsLondon { effectiveTip = cmath.BigMin(st.gasTipCap, new(big.Int).Sub(st.gasFeeCap, st.evm.Context.BaseFee)) } - st.state.AddBalance(st.evm.Context.Coinbase, new(big.Int).Mul(new(big.Int).SetUint64(st.gasUsed()), effectiveTip)) + + if st.evm.Config.NoBaseFee && st.gasFeeCap.Sign() == 0 && st.gasTipCap.Sign() == 0 { + // Skip fee payment when NoBaseFee is set and the fee fields + // are 0. This avoids a negative effectiveTip being applied to + // the coinbase when simulating calls. + } else { + fee := new(big.Int).SetUint64(st.gasUsed()) + fee.Mul(fee, effectiveTip) + st.state.AddBalance(st.evm.Context.Coinbase, fee) + } return &ExecutionResult{ UsedGas: st.gasUsed(), diff --git a/core/tx_journal.go b/core/tx_journal.go index 5453ee191658d..62344f5646762 100644 --- a/core/tx_journal.go +++ b/core/tx_journal.go @@ -19,6 +19,7 @@ package core import ( "errors" "io" + "io/fs" "os" "github.com/ethereum/go-ethereum/common" @@ -57,12 +58,12 @@ func newTxJournal(path string) *txJournal { // load parses a transaction journal dump from disk, loading its contents into // the specified pool. func (journal *txJournal) load(add func([]*types.Transaction) []error) error { - // Skip the parsing if the journal file doesn't exist at all - if !common.FileExist(journal.path) { - return nil - } // Open the journal for loading any past transactions input, err := os.Open(journal.path) + if errors.Is(err, fs.ErrNotExist) { + // Skip the parsing if the journal file doesn't exist at all + return nil + } if err != nil { return err } diff --git a/core/tx_pool.go b/core/tx_pool.go index 81a726bae4abc..1c25442dd9c59 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -1459,7 +1459,7 @@ func (pool *TxPool) truncatePending() { pendingRateLimitMeter.Mark(int64(pendingBeforeCap - pending)) } -// truncateQueue drops the oldes transactions in the queue if the pool is above the global queue limit. +// truncateQueue drops the oldest transactions in the queue if the pool is above the global queue limit. func (pool *TxPool) truncateQueue() { queued := uint64(0) for _, list := range pool.queue { @@ -1603,10 +1603,6 @@ func (as *accountSet) contains(addr common.Address) bool { return exist } -func (as *accountSet) empty() bool { - return len(as.accounts) == 0 -} - // containsTx checks if the sender of a given tx is within the set. If the sender // cannot be derived, this method returns false. func (as *accountSet) containsTx(tx *types.Transaction) bool { diff --git a/core/tx_pool_test.go b/core/tx_pool_test.go index dd2407470daa5..2fd0f529f8f2c 100644 --- a/core/tx_pool_test.go +++ b/core/tx_pool_test.go @@ -669,7 +669,6 @@ func TestTransactionPostponing(t *testing.T) { // Add a batch consecutive pending transactions for validation txs := []*types.Transaction{} for i, key := range keys { - for j := 0; j < 100; j++ { var tx *types.Transaction if (i+j)%2 == 0 { diff --git a/core/types/block.go b/core/types/block.go index 8386c4c440e81..7525a88f5a385 100644 --- a/core/types/block.go +++ b/core/types/block.go @@ -172,10 +172,6 @@ type Block struct { hash atomic.Value size atomic.Value - // Td is used by package core to store the total difficulty - // of the chain up to and including the block. - td *big.Int - // These fields are used by package eth to track // inter-peer block relay. ReceivedAt time.Time @@ -197,7 +193,7 @@ type extblock struct { // are ignored and set to values derived from the given txs, uncles // and receipts. func NewBlock(header *Header, txs []*Transaction, uncles []*Header, receipts []*Receipt, hasher TrieHasher) *Block { - b := &Block{header: CopyHeader(header), td: new(big.Int)} + b := &Block{header: CopyHeader(header)} // TODO: panic if len(txs) != len(receipts) if len(txs) == 0 { @@ -321,7 +317,7 @@ func (b *Block) Header() *Header { return CopyHeader(b.header) } func (b *Block) Body() *Body { return &Body{b.transactions, b.uncles} } // Size returns the true RLP encoded storage size of the block, either by encoding -// and returning it, or returning a previsouly cached value. +// and returning it, or returning a previously cached value. func (b *Block) Size() common.StorageSize { if size := b.size.Load(); size != nil { return size.(common.StorageSize) diff --git a/core/types/block_test.go b/core/types/block_test.go index aa1db2f4faade..9e7f581b1dc4d 100644 --- a/core/types/block_test.go +++ b/core/types/block_test.go @@ -314,7 +314,7 @@ func TestRlpDecodeParentHash(t *testing.T) { } // Also test a very very large header. { - // The rlp-encoding of the heder belowCauses _total_ length of 65540, + // The rlp-encoding of the header belowCauses _total_ length of 65540, // which is the first to blow the fast-path. h := &Header{ ParentHash: want, diff --git a/core/types/bloom9.go b/core/types/bloom9.go index 1793c2adc73c9..a560a20724fd7 100644 --- a/core/types/bloom9.go +++ b/core/types/bloom9.go @@ -154,7 +154,7 @@ func bloomValues(data []byte, hashbuf []byte) (uint, byte, uint, byte, uint, byt return i1, v1, i2, v2, i3, v3 } -// BloomLookup is a convenience-method to check presence int he bloom filter +// BloomLookup is a convenience-method to check presence in the bloom filter func BloomLookup(bin Bloom, topic bytesBacked) bool { return bin.Test(topic.Bytes()) } diff --git a/core/types/bloom9_test.go b/core/types/bloom9_test.go index 893df486dd1bb..d3178d112efb9 100644 --- a/core/types/bloom9_test.go +++ b/core/types/bloom9_test.go @@ -92,7 +92,6 @@ func BenchmarkBloom9Lookup(b *testing.B) { } func BenchmarkCreateBloom(b *testing.B) { - var txs = Transactions{ NewContractCreation(1, big.NewInt(1), 1, big.NewInt(1), nil), NewTransaction(2, common.HexToAddress("0x2"), big.NewInt(2), 2, big.NewInt(2), nil), diff --git a/core/types/hashing.go b/core/types/hashing.go index a115a8842ec3b..3df75432a4b43 100644 --- a/core/types/hashing.go +++ b/core/types/hashing.go @@ -31,7 +31,7 @@ var hasherPool = sync.Pool{ New: func() interface{} { return sha3.NewLegacyKeccak256() }, } -// deriveBufferPool holds temporary encoder buffers for DeriveSha and TX encoding. +// encodeBufferPool holds temporary encoder buffers for DeriveSha and TX encoding. var encodeBufferPool = sync.Pool{ New: func() interface{} { return new(bytes.Buffer) }, } diff --git a/core/types/hashing_test.go b/core/types/hashing_test.go index de71ee41a47d7..294a3977d03b0 100644 --- a/core/types/hashing_test.go +++ b/core/types/hashing_test.go @@ -39,8 +39,7 @@ func TestDeriveSha(t *testing.T) { t.Fatal(err) } for len(txs) < 1000 { - tr, _ := trie.New(common.Hash{}, trie.NewDatabase(rawdb.NewMemoryDatabase())) - exp := types.DeriveSha(txs, tr) + exp := types.DeriveSha(txs, trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase()))) got := types.DeriveSha(txs, trie.NewStackTrie(nil)) if !bytes.Equal(got[:], exp[:]) { t.Fatalf("%d txs: got %x exp %x", len(txs), got, exp) @@ -87,8 +86,7 @@ func BenchmarkDeriveSha200(b *testing.B) { b.ResetTimer() b.ReportAllocs() for i := 0; i < b.N; i++ { - tr, _ := trie.New(common.Hash{}, trie.NewDatabase(rawdb.NewMemoryDatabase())) - exp = types.DeriveSha(txs, tr) + exp = types.DeriveSha(txs, trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase()))) } }) @@ -109,8 +107,7 @@ func TestFuzzDeriveSha(t *testing.T) { rndSeed := mrand.Int() for i := 0; i < 10; i++ { seed := rndSeed + i - tr, _ := trie.New(common.Hash{}, trie.NewDatabase(rawdb.NewMemoryDatabase())) - exp := types.DeriveSha(newDummy(i), tr) + exp := types.DeriveSha(newDummy(i), trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase()))) got := types.DeriveSha(newDummy(i), trie.NewStackTrie(nil)) if !bytes.Equal(got[:], exp[:]) { printList(newDummy(seed)) @@ -138,8 +135,7 @@ func TestDerivableList(t *testing.T) { }, } for i, tc := range tcs[1:] { - tr, _ := trie.New(common.Hash{}, trie.NewDatabase(rawdb.NewMemoryDatabase())) - exp := types.DeriveSha(flatList(tc), tr) + exp := types.DeriveSha(flatList(tc), trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase()))) got := types.DeriveSha(flatList(tc), trie.NewStackTrie(nil)) if !bytes.Equal(got[:], exp[:]) { t.Fatalf("case %d: got %x exp %x", i, got, exp) @@ -201,7 +197,7 @@ func printList(l types.DerivableList) { for i := 0; i < l.Len(); i++ { var buf bytes.Buffer l.EncodeIndex(i, &buf) - fmt.Printf("\"0x%x\",\n", buf.Bytes()) + fmt.Printf("\"%#x\",\n", buf.Bytes()) } fmt.Printf("},\n") } diff --git a/core/types/log.go b/core/types/log.go index ee323ba868041..eb30957b12788 100644 --- a/core/types/log.go +++ b/core/types/log.go @@ -98,8 +98,8 @@ func (l *Log) DecodeRLP(s *rlp.Stream) error { return err } -// LogForStorage is a wrapper around a Log that flattens and parses the entire content of -// a log including non-consensus fields. +// LogForStorage is a wrapper around a Log that handles +// backward compatibility with prior storage formats. type LogForStorage Log // EncodeRLP implements rlp.Encoder. diff --git a/core/types/receipt.go b/core/types/receipt.go index a913cd0e83be9..bdf48451473c3 100644 --- a/core/types/receipt.go +++ b/core/types/receipt.go @@ -267,8 +267,8 @@ func (r *Receipt) Size() common.StorageSize { return size } -// ReceiptForStorage is a wrapper around a Receipt that flattens and parses the -// entire content of a receipt, as opposed to only the consensus fields originally. +// ReceiptForStorage is a wrapper around a Receipt with RLP serialization +// that omits the Bloom field and deserialization that re-computes it. type ReceiptForStorage Receipt // EncodeRLP implements rlp.Encoder, and flattens all content fields of a receipt diff --git a/core/types/transaction.go b/core/types/transaction.go index 29820a0d785fc..715ede15db2eb 100644 --- a/core/types/transaction.go +++ b/core/types/transaction.go @@ -432,6 +432,24 @@ func TxDifference(a, b Transactions) Transactions { return keep } +// HashDifference returns a new set which is the difference between a and b. +func HashDifference(a, b []common.Hash) []common.Hash { + keep := make([]common.Hash, 0, len(a)) + + remove := make(map[common.Hash]struct{}) + for _, hash := range b { + remove[hash] = struct{}{} + } + + for _, hash := range a { + if _, ok := remove[hash]; !ok { + keep = append(keep, hash) + } + } + + return keep +} + // TxByNonce implements the sort interface to allow sorting a list of transactions // by their nonces. This is usually only useful for sorting transactions from a // single account, otherwise a nonce comparison doesn't make much sense. diff --git a/core/types/transaction_signing_test.go b/core/types/transaction_signing_test.go index 689fc38a9b660..1c775f129d654 100644 --- a/core/types/transaction_signing_test.go +++ b/core/types/transaction_signing_test.go @@ -111,7 +111,6 @@ func TestEIP155SigningVitalik(t *testing.T) { if from != addr { t.Errorf("%d: expected %x got %x", i, addr, from) } - } } diff --git a/core/types/transaction_test.go b/core/types/transaction_test.go index 2e418b2309861..67e5b3cce3f5b 100644 --- a/core/types/transaction_test.go +++ b/core/types/transaction_test.go @@ -114,7 +114,6 @@ func TestEIP2718TransactionSigHash(t *testing.T) { // This test checks signature operations on access list transactions. func TestEIP2930Signer(t *testing.T) { - var ( key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") keyAddr = crypto.PubkeyToAddress(key.PublicKey) diff --git a/core/vm/analysis.go b/core/vm/analysis.go index 3733bab6a7c07..4aa8cfe70f11d 100644 --- a/core/vm/analysis.go +++ b/core/vm/analysis.go @@ -76,7 +76,7 @@ func codeBitmapInternal(code, bits bitvec) bitvec { for pc := uint64(0); pc < uint64(len(code)); { op := OpCode(code[pc]) pc++ - if op < PUSH1 || op > PUSH32 { + if int8(op) < int8(PUSH1) { // If not PUSH (the int8(op) > int(PUSH32) is always false). continue } numbits := op - PUSH1 + 1 diff --git a/core/vm/contracts.go b/core/vm/contracts.go index 79f1a3611680f..1b832b638695b 100644 --- a/core/vm/contracts.go +++ b/core/vm/contracts.go @@ -29,8 +29,6 @@ import ( "github.com/ethereum/go-ethereum/crypto/bls12381" "github.com/ethereum/go-ethereum/crypto/bn256" "github.com/ethereum/go-ethereum/params" - - //lint:ignore SA1019 Needed for precompile "golang.org/x/crypto/ripemd160" ) diff --git a/core/vm/instructions_test.go b/core/vm/instructions_test.go index f0fa4811ca7b5..602cde51015ea 100644 --- a/core/vm/instructions_test.go +++ b/core/vm/instructions_test.go @@ -46,7 +46,6 @@ var commonParams []*twoOperandParams var twoOpMethods map[string]executionFunc func init() { - // Params is a list of common edgecases that should be used for some common tests params := []string{ "0000000000000000000000000000000000000000000000000000000000000000", // 0 @@ -92,7 +91,6 @@ func init() { } func testTwoOperandOp(t *testing.T, tests []TwoOperandTestcase, opFn executionFunc, name string) { - var ( env = NewEVM(BlockContext{}, TxContext{}, nil, params.TestChainConfig, Config{}) stack = newstack() @@ -229,32 +227,32 @@ func TestAddMod(t *testing.T) { } } -// getResult is a convenience function to generate the expected values -func getResult(args []*twoOperandParams, opFn executionFunc) []TwoOperandTestcase { - var ( - env = NewEVM(BlockContext{}, TxContext{}, nil, params.TestChainConfig, Config{}) - stack = newstack() - pc = uint64(0) - interpreter = env.interpreter - ) - result := make([]TwoOperandTestcase, len(args)) - for i, param := range args { - x := new(uint256.Int).SetBytes(common.Hex2Bytes(param.x)) - y := new(uint256.Int).SetBytes(common.Hex2Bytes(param.y)) - stack.push(x) - stack.push(y) - opFn(&pc, interpreter, &ScopeContext{nil, stack, nil}) - actual := stack.pop() - result[i] = TwoOperandTestcase{param.x, param.y, fmt.Sprintf("%064x", actual)} - } - return result -} - // utility function to fill the json-file with testcases // Enable this test to generate the 'testcases_xx.json' files func TestWriteExpectedValues(t *testing.T) { t.Skip("Enable this test to create json test cases.") + // getResult is a convenience function to generate the expected values + getResult := func(args []*twoOperandParams, opFn executionFunc) []TwoOperandTestcase { + var ( + env = NewEVM(BlockContext{}, TxContext{}, nil, params.TestChainConfig, Config{}) + stack = newstack() + pc = uint64(0) + interpreter = env.interpreter + ) + result := make([]TwoOperandTestcase, len(args)) + for i, param := range args { + x := new(uint256.Int).SetBytes(common.Hex2Bytes(param.x)) + y := new(uint256.Int).SetBytes(common.Hex2Bytes(param.y)) + stack.push(x) + stack.push(y) + opFn(&pc, interpreter, &ScopeContext{nil, stack, nil}) + actual := stack.pop() + result[i] = TwoOperandTestcase{param.x, param.y, fmt.Sprintf("%064x", actual)} + } + return result + } + for name, method := range twoOpMethods { data, err := json.Marshal(getResult(commonParams, method)) if err != nil { @@ -641,7 +639,6 @@ func TestCreate2Addreses(t *testing.T) { expected: "0xE33C0C7F7df4809055C3ebA6c09CFe4BaF1BD9e0", }, } { - origin := common.BytesToAddress(common.FromHex(tt.origin)) salt := common.BytesToHash(common.FromHex(tt.salt)) code := common.FromHex(tt.code) diff --git a/core/vm/interpreter.go b/core/vm/interpreter.go index 4f1ebc43a229f..40fe23dc516ce 100644 --- a/core/vm/interpreter.go +++ b/core/vm/interpreter.go @@ -114,7 +114,6 @@ func NewEVMInterpreter(evm *EVM, cfg Config) *EVMInterpreter { // considered a revert-and-consume-all-gas operation except for // ErrExecutionReverted which means revert-and-keep-gas-left. func (in *EVMInterpreter) Run(contract *Contract, input []byte, readOnly bool) (ret []byte, err error) { - // Increment the call depth which is restricted to 1024 in.evm.depth++ defer func() { in.evm.depth-- }() diff --git a/core/vm/interpreter_test.go b/core/vm/interpreter_test.go index dfae0f2e2af7d..31ee9922dbac3 100644 --- a/core/vm/interpreter_test.go +++ b/core/vm/interpreter_test.go @@ -73,5 +73,4 @@ func TestLoopInterrupt(t *testing.T) { } } } - } diff --git a/core/vm/jump_table.go b/core/vm/jump_table.go index eef3b53d8c668..707b52e79dafa 100644 --- a/core/vm/jump_table.go +++ b/core/vm/jump_table.go @@ -63,7 +63,7 @@ type JumpTable [256]*operation func validate(jt JumpTable) JumpTable { for i, op := range jt { if op == nil { - panic(fmt.Sprintf("op 0x%x is not set", i)) + panic(fmt.Sprintf("op %#x is not set", i)) } // The interpreter has an assumption that if the memorySize function is // set, then the dynamicGas function is also set. This is a somewhat @@ -198,7 +198,6 @@ func newSpuriousDragonInstructionSet() JumpTable { instructionSet := newTangerineWhistleInstructionSet() instructionSet[EXP].dynamicGas = gasExpEIP158 return validate(instructionSet) - } // EIP 150 a.k.a Tangerine Whistle diff --git a/core/vm/opcodes.go b/core/vm/opcodes.go index 19252b01f2569..77d619abb9c1b 100644 --- a/core/vm/opcodes.go +++ b/core/vm/opcodes.go @@ -392,7 +392,7 @@ var opCodeToString = map[OpCode]string{ func (op OpCode) String() string { str := opCodeToString[op] if len(str) == 0 { - return fmt.Sprintf("opcode 0x%x not defined", int(op)) + return fmt.Sprintf("opcode %#x not defined", int(op)) } return str diff --git a/core/vm/runtime/runtime_test.go b/core/vm/runtime/runtime_test.go index fcaa10f1c62cc..ab77e284df35f 100644 --- a/core/vm/runtime/runtime_test.go +++ b/core/vm/runtime/runtime_test.go @@ -22,7 +22,6 @@ import ( "os" "strings" "testing" - "time" "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" @@ -326,25 +325,6 @@ func TestBlockhash(t *testing.T) { } } -type stepCounter struct { - inner *logger.JSONLogger - steps int -} - -func (s *stepCounter) CaptureStart(env *vm.EVM, from common.Address, to common.Address, create bool, input []byte, gas uint64, value *big.Int) { -} - -func (s *stepCounter) CaptureFault(pc uint64, op vm.OpCode, gas, cost uint64, scope *vm.ScopeContext, depth int, err error) { -} - -func (s *stepCounter) CaptureEnd(output []byte, gasUsed uint64, t time.Duration, err error) {} - -func (s *stepCounter) CaptureState(pc uint64, op vm.OpCode, gas, cost uint64, scope *vm.ScopeContext, rData []byte, depth int, err error) { - s.steps++ - // Enable this for more output - //s.inner.CaptureState(env, pc, op, gas, cost, memory, stack, rStack, contract, depth, err) -} - // benchmarkNonModifyingCode benchmarks code, but if the code modifies the // state, this should not be used, since it does not reset the state between runs. func benchmarkNonModifyingCode(gas uint64, code []byte, name string, tracerCode string, b *testing.B) { @@ -353,7 +333,7 @@ func benchmarkNonModifyingCode(gas uint64, code []byte, name string, tracerCode cfg.State, _ = state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) cfg.GasLimit = gas if len(tracerCode) > 0 { - tracer, err := tracers.New(tracerCode, new(tracers.Context)) + tracer, err := tracers.New(tracerCode, new(tracers.Context), nil) if err != nil { b.Fatal(err) } @@ -399,7 +379,6 @@ func benchmarkNonModifyingCode(gas uint64, code []byte, name string, tracerCode // BenchmarkSimpleLoop test a pretty simple loop which loops until OOG // 55 ms func BenchmarkSimpleLoop(b *testing.B) { - staticCallIdentity := []byte{ byte(vm.JUMPDEST), // [ count ] // push args for the call @@ -478,7 +457,7 @@ func BenchmarkSimpleLoop(b *testing.B) { byte(vm.JUMP), } - calllRevertingContractWithInput := []byte{ + callRevertingContractWithInput := []byte{ byte(vm.JUMPDEST), // // push args for the call byte(vm.PUSH1), 0, // out size @@ -506,7 +485,7 @@ func BenchmarkSimpleLoop(b *testing.B) { benchmarkNonModifyingCode(100000000, loopingCode, "loop-100M", "", b) benchmarkNonModifyingCode(100000000, callInexistant, "call-nonexist-100M", "", b) benchmarkNonModifyingCode(100000000, callEOA, "call-EOA-100M", "", b) - benchmarkNonModifyingCode(100000000, calllRevertingContractWithInput, "call-reverting-100M", "", b) + benchmarkNonModifyingCode(100000000, callRevertingContractWithInput, "call-reverting-100M", "", b) //benchmarkNonModifyingCode(10000000, staticCallIdentity, "staticcall-identity-10M", b) //benchmarkNonModifyingCode(10000000, loopingCode, "loop-10M", b) @@ -518,12 +497,11 @@ func TestEip2929Cases(t *testing.T) { t.Skip("Test only useful for generating documentation") id := 1 prettyPrint := func(comment string, code []byte) { - instrs := make([]string, 0) it := asm.NewInstructionIterator(code) for it.Next() { if it.Arg() != nil && 0 < len(it.Arg()) { - instrs = append(instrs, fmt.Sprintf("%v 0x%x", it.Op(), it.Arg())) + instrs = append(instrs, fmt.Sprintf("%v %#x", it.Op(), it.Arg())) } else { instrs = append(instrs, fmt.Sprintf("%v", it.Op())) } @@ -531,7 +509,7 @@ func TestEip2929Cases(t *testing.T) { ops := strings.Join(instrs, ", ") fmt.Printf("### Case %d\n\n", id) id++ - fmt.Printf("%v\n\nBytecode: \n```\n0x%x\n```\nOperations: \n```\n%v\n```\n\n", + fmt.Printf("%v\n\nBytecode: \n```\n%#x\n```\nOperations: \n```\n%v\n```\n\n", comment, code, ops) Execute(code, nil, &Config{ @@ -854,7 +832,7 @@ func TestRuntimeJSTracer(t *testing.T) { statedb.SetCode(common.HexToAddress("0xee"), calleeCode) statedb.SetCode(common.HexToAddress("0xff"), depressedCode) - tracer, err := tracers.New(jsTracer, new(tracers.Context)) + tracer, err := tracers.New(jsTracer, new(tracers.Context), nil) if err != nil { t.Fatal(err) } @@ -890,7 +868,7 @@ func TestJSTracerCreateTx(t *testing.T) { code := []byte{byte(vm.PUSH1), 0, byte(vm.PUSH1), 0, byte(vm.RETURN)} statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) - tracer, err := tracers.New(jsTracer, new(tracers.Context)) + tracer, err := tracers.New(jsTracer, new(tracers.Context), nil) if err != nil { t.Fatal(err) } diff --git a/crypto/blake2b/blake2b.go b/crypto/blake2b/blake2b.go index 5da50cab6f00e..7ecaab813999f 100644 --- a/crypto/blake2b/blake2b.go +++ b/crypto/blake2b/blake2b.go @@ -302,6 +302,7 @@ func appendUint64(b []byte, x uint64) []byte { return append(b, a[:]...) } +//nolint:unused,deadcode func appendUint32(b []byte, x uint32) []byte { var a [4]byte binary.BigEndian.PutUint32(a[:], x) @@ -313,6 +314,7 @@ func consumeUint64(b []byte) ([]byte, uint64) { return b[8:], x } +//nolint:unused,deadcode func consumeUint32(b []byte) ([]byte, uint32) { x := binary.BigEndian.Uint32(b) return b[4:], x diff --git a/crypto/blake2b/blake2b_generic.go b/crypto/blake2b/blake2b_generic.go index 35c40cc924f8d..61e678fdf5763 100644 --- a/crypto/blake2b/blake2b_generic.go +++ b/crypto/blake2b/blake2b_generic.go @@ -25,6 +25,7 @@ var precomputed = [10][16]byte{ {10, 8, 7, 1, 2, 4, 6, 5, 15, 9, 3, 13, 11, 14, 12, 0}, } +// nolint:unused,deadcode func hashBlocksGeneric(h *[8]uint64, c *[2]uint64, flag uint64, blocks []byte) { var m [16]uint64 c0, c1 := c[0], c[1] diff --git a/crypto/blake2b/blake2b_test.go b/crypto/blake2b/blake2b_test.go index 9e7297da160f3..9d24444a27b7e 100644 --- a/crypto/blake2b/blake2b_test.go +++ b/crypto/blake2b/blake2b_test.go @@ -14,14 +14,6 @@ import ( "testing" ) -func fromHex(s string) []byte { - b, err := hex.DecodeString(s) - if err != nil { - panic(err) - } - return b -} - func TestHashes(t *testing.T) { defer func(sse4, avx, avx2 bool) { useSSE4, useAVX, useAVX2 = sse4, avx, avx2 diff --git a/crypto/bls12381/field_element_test.go b/crypto/bls12381/field_element_test.go index 0f6abd280cbb0..70bbe5cfe5e71 100644 --- a/crypto/bls12381/field_element_test.go +++ b/crypto/bls12381/field_element_test.go @@ -102,7 +102,6 @@ func TestFieldElementEquality(t *testing.T) { if a12.equal(b12) { t.Fatal("a != a + 1") } - } func TestFieldElementHelpers(t *testing.T) { diff --git a/crypto/bls12381/fp12.go b/crypto/bls12381/fp12.go index 3141c76c3995a..51e949fe5f049 100644 --- a/crypto/bls12381/fp12.go +++ b/crypto/bls12381/fp12.go @@ -96,7 +96,6 @@ func (e *fp12) add(c, a, b *fe12) { fp6 := e.fp6 fp6.add(&c[0], &a[0], &b[0]) fp6.add(&c[1], &a[1], &b[1]) - } func (e *fp12) double(c, a *fe12) { @@ -109,7 +108,6 @@ func (e *fp12) sub(c, a, b *fe12) { fp6 := e.fp6 fp6.sub(&c[0], &a[0], &b[0]) fp6.sub(&c[1], &a[1], &b[1]) - } func (e *fp12) neg(c, a *fe12) { diff --git a/crypto/bls12381/fp_test.go b/crypto/bls12381/fp_test.go index 97528d9db32ea..0bad35de16304 100644 --- a/crypto/bls12381/fp_test.go +++ b/crypto/bls12381/fp_test.go @@ -465,7 +465,6 @@ func TestFpNonResidue(t *testing.T) { i -= 1 } } - } func TestFp2Serialization(t *testing.T) { diff --git a/crypto/bls12381/g2.go b/crypto/bls12381/g2.go index fa110e3edfc5b..c2ca959bcca1b 100644 --- a/crypto/bls12381/g2.go +++ b/crypto/bls12381/g2.go @@ -41,7 +41,6 @@ func (p *PointG2) Zero() *PointG2 { p[1].one() p[2].zero() return p - } type tempG2 struct { diff --git a/crypto/bls12381/isogeny.go b/crypto/bls12381/isogeny.go index c3cb0a6f7bf07..a63f585dd00a0 100644 --- a/crypto/bls12381/isogeny.go +++ b/crypto/bls12381/isogeny.go @@ -19,7 +19,7 @@ package bls12381 // isogenyMapG1 applies 11-isogeny map for BLS12-381 G1 defined at draft-irtf-cfrg-hash-to-curve-06. func isogenyMapG1(x, y *fe) { // https://tools.ietf.org/html/draft-irtf-cfrg-hash-to-curve-06#appendix-C.2 - params := isogenyConstansG1 + params := isogenyConstantsG1 degree := 15 xNum, xDen, yNum, yDen := new(fe), new(fe), new(fe), new(fe) xNum.set(params[0][degree]) @@ -76,7 +76,7 @@ func isogenyMapG2(e *fp2, x, y *fe2) { y.set(yNum) } -var isogenyConstansG1 = [4][16]*fe{ +var isogenyConstantsG1 = [4][16]*fe{ { {0x4d18b6f3af00131c, 0x19fa219793fee28c, 0x3f2885f1467f19ae, 0x23dcea34f2ffb304, 0xd15b58d2ffc00054, 0x0913be200a20bef4}, {0x898985385cdbbd8b, 0x3c79e43cc7d966aa, 0x1597e193f4cd233a, 0x8637ef1e4d6623ad, 0x11b22deed20d827b, 0x07097bc5998784ad}, diff --git a/crypto/bn256/cloudflare/gfp_decl.go b/crypto/bn256/cloudflare/gfp_decl.go index ec4018e88a0c0..cf7f5654239fa 100644 --- a/crypto/bn256/cloudflare/gfp_decl.go +++ b/crypto/bn256/cloudflare/gfp_decl.go @@ -10,7 +10,7 @@ import ( "golang.org/x/sys/cpu" ) -//nolint:varcheck +//nolint:varcheck,unused,deadcode var hasBMI2 = cpu.X86.HasBMI2 // go:noescape diff --git a/crypto/ecies/ecies_test.go b/crypto/ecies/ecies_test.go index 96e33da006fb8..8ca42c9c8ee63 100644 --- a/crypto/ecies/ecies_test.go +++ b/crypto/ecies/ecies_test.go @@ -334,7 +334,6 @@ func testParamSelection(t *testing.T, c testCase) { if err == nil { t.Fatalf("ecies: encryption should not have succeeded (%s)\n", c.Name) } - } // Ensure that the basic public key validation in the decryption operation diff --git a/eth/api.go b/eth/api.go index ef69acb76eb45..5642ef4c3f127 100644 --- a/eth/api.go +++ b/eth/api.go @@ -41,57 +41,44 @@ import ( "github.com/ethereum/go-ethereum/trie" ) -// PublicEthereumAPI provides an API to access Ethereum full node-related -// information. -type PublicEthereumAPI struct { +// EthereumAPI provides an API to access Ethereum full node-related information. +type EthereumAPI struct { e *Ethereum } -// NewPublicEthereumAPI creates a new Ethereum protocol API for full nodes. -func NewPublicEthereumAPI(e *Ethereum) *PublicEthereumAPI { - return &PublicEthereumAPI{e} +// NewEthereumAPI creates a new Ethereum protocol API for full nodes. +func NewEthereumAPI(e *Ethereum) *EthereumAPI { + return &EthereumAPI{e} } -// Etherbase is the address that mining rewards will be send to -func (api *PublicEthereumAPI) Etherbase() (common.Address, error) { +// Etherbase is the address that mining rewards will be send to. +func (api *EthereumAPI) Etherbase() (common.Address, error) { return api.e.Etherbase() } -// Coinbase is the address that mining rewards will be send to (alias for Etherbase) -func (api *PublicEthereumAPI) Coinbase() (common.Address, error) { +// Coinbase is the address that mining rewards will be send to (alias for Etherbase). +func (api *EthereumAPI) Coinbase() (common.Address, error) { return api.Etherbase() } -// Hashrate returns the POW hashrate -func (api *PublicEthereumAPI) Hashrate() hexutil.Uint64 { +// Hashrate returns the POW hashrate. +func (api *EthereumAPI) Hashrate() hexutil.Uint64 { return hexutil.Uint64(api.e.Miner().Hashrate()) } -// PublicMinerAPI provides an API to control the miner. -// It offers only methods that operate on data that pose no security risk when it is publicly accessible. -type PublicMinerAPI struct { - e *Ethereum -} - -// NewPublicMinerAPI create a new PublicMinerAPI instance. -func NewPublicMinerAPI(e *Ethereum) *PublicMinerAPI { - return &PublicMinerAPI{e} -} - // Mining returns an indication if this node is currently mining. -func (api *PublicMinerAPI) Mining() bool { +func (api *EthereumAPI) Mining() bool { return api.e.IsMining() } -// PrivateMinerAPI provides private RPC methods to control the miner. -// These methods can be abused by external users and must be considered insecure for use by untrusted users. -type PrivateMinerAPI struct { +// MinerAPI provides an API to control the miner. +type MinerAPI struct { e *Ethereum } -// NewPrivateMinerAPI create a new RPC service which controls the miner of this node. -func NewPrivateMinerAPI(e *Ethereum) *PrivateMinerAPI { - return &PrivateMinerAPI{e: e} +// NewMinerAPI create a new MinerAPI instance. +func NewMinerAPI(e *Ethereum) *MinerAPI { + return &MinerAPI{e} } // Start starts the miner with the given number of threads. If threads is nil, @@ -99,7 +86,7 @@ func NewPrivateMinerAPI(e *Ethereum) *PrivateMinerAPI { // usable by this process. If mining is already running, this method adjust the // number of threads allowed to use and updates the minimum price required by the // transaction pool. -func (api *PrivateMinerAPI) Start(threads *int) error { +func (api *MinerAPI) Start(threads *int) error { if threads == nil { return api.e.StartMining(runtime.NumCPU()) } @@ -108,12 +95,12 @@ func (api *PrivateMinerAPI) Start(threads *int) error { // Stop terminates the miner, both at the consensus engine level as well as at // the block creation level. -func (api *PrivateMinerAPI) Stop() { +func (api *MinerAPI) Stop() { api.e.StopMining() } // SetExtra sets the extra data string that is included when this miner mines a block. -func (api *PrivateMinerAPI) SetExtra(extra string) (bool, error) { +func (api *MinerAPI) SetExtra(extra string) (bool, error) { if err := api.e.Miner().SetExtra([]byte(extra)); err != nil { return false, err } @@ -121,7 +108,7 @@ func (api *PrivateMinerAPI) SetExtra(extra string) (bool, error) { } // SetGasPrice sets the minimum accepted gas price for the miner. -func (api *PrivateMinerAPI) SetGasPrice(gasPrice hexutil.Big) bool { +func (api *MinerAPI) SetGasPrice(gasPrice hexutil.Big) bool { api.e.lock.Lock() api.e.gasPrice = (*big.Int)(&gasPrice) api.e.lock.Unlock() @@ -131,37 +118,36 @@ func (api *PrivateMinerAPI) SetGasPrice(gasPrice hexutil.Big) bool { } // SetGasLimit sets the gaslimit to target towards during mining. -func (api *PrivateMinerAPI) SetGasLimit(gasLimit hexutil.Uint64) bool { +func (api *MinerAPI) SetGasLimit(gasLimit hexutil.Uint64) bool { api.e.Miner().SetGasCeil(uint64(gasLimit)) return true } -// SetEtherbase sets the etherbase of the miner -func (api *PrivateMinerAPI) SetEtherbase(etherbase common.Address) bool { +// SetEtherbase sets the etherbase of the miner. +func (api *MinerAPI) SetEtherbase(etherbase common.Address) bool { api.e.SetEtherbase(etherbase) return true } // SetRecommitInterval updates the interval for miner sealing work recommitting. -func (api *PrivateMinerAPI) SetRecommitInterval(interval int) { +func (api *MinerAPI) SetRecommitInterval(interval int) { api.e.Miner().SetRecommitInterval(time.Duration(interval) * time.Millisecond) } -// PrivateAdminAPI is the collection of Ethereum full node-related APIs -// exposed over the private admin endpoint. -type PrivateAdminAPI struct { +// AdminAPI is the collection of Ethereum full node related APIs for node +// administration. +type AdminAPI struct { eth *Ethereum } -// NewPrivateAdminAPI creates a new API definition for the full node private -// admin methods of the Ethereum service. -func NewPrivateAdminAPI(eth *Ethereum) *PrivateAdminAPI { - return &PrivateAdminAPI{eth: eth} +// NewAdminAPI creates a new instance of AdminAPI. +func NewAdminAPI(eth *Ethereum) *AdminAPI { + return &AdminAPI{eth: eth} } // ExportChain exports the current blockchain into a local file, -// or a range of blocks if first and last are non-nil -func (api *PrivateAdminAPI) ExportChain(file string, first *uint64, last *uint64) (bool, error) { +// or a range of blocks if first and last are non-nil. +func (api *AdminAPI) ExportChain(file string, first *uint64, last *uint64) (bool, error) { if first == nil && last != nil { return false, errors.New("last cannot be specified without first") } @@ -171,7 +157,7 @@ func (api *PrivateAdminAPI) ExportChain(file string, first *uint64, last *uint64 } if _, err := os.Stat(file); err == nil { // File already exists. Allowing overwrite could be a DoS vector, - // since the 'file' may point to arbitrary paths on the drive + // since the 'file' may point to arbitrary paths on the drive. return false, errors.New("location would overwrite an existing file") } // Make sure we can create the file to export into @@ -209,7 +195,7 @@ func hasAllBlocks(chain *core.BlockChain, bs []*types.Block) bool { } // ImportChain imports a blockchain from a local file. -func (api *PrivateAdminAPI) ImportChain(file string) (bool, error) { +func (api *AdminAPI) ImportChain(file string) (bool, error) { // Make sure the can access the file to import in, err := os.Open(file) if err != nil { @@ -257,20 +243,19 @@ func (api *PrivateAdminAPI) ImportChain(file string) (bool, error) { return true, nil } -// PublicDebugAPI is the collection of Ethereum full node APIs exposed -// over the public debugging endpoint. -type PublicDebugAPI struct { +// DebugAPI is the collection of Ethereum full node APIs for debugging the +// protocol. +type DebugAPI struct { eth *Ethereum } -// NewPublicDebugAPI creates a new API definition for the full node- -// related public debug methods of the Ethereum service. -func NewPublicDebugAPI(eth *Ethereum) *PublicDebugAPI { - return &PublicDebugAPI{eth: eth} +// NewDebugAPI creates a new DebugAPI instance. +func NewDebugAPI(eth *Ethereum) *DebugAPI { + return &DebugAPI{eth: eth} } // DumpBlock retrieves the entire state of the database at a given block. -func (api *PublicDebugAPI) DumpBlock(blockNr rpc.BlockNumber) (state.Dump, error) { +func (api *DebugAPI) DumpBlock(blockNr rpc.BlockNumber) (state.Dump, error) { opts := &state.DumpConfig{ OnlyWithAddresses: true, Max: AccountRangeMaxResults, // Sanity limit over RPC @@ -287,6 +272,8 @@ func (api *PublicDebugAPI) DumpBlock(blockNr rpc.BlockNumber) (state.Dump, error block = api.eth.blockchain.CurrentBlock() } else if blockNr == rpc.FinalizedBlockNumber { block = api.eth.blockchain.CurrentFinalizedBlock() + } else if blockNr == rpc.SafeBlockNumber { + block = api.eth.blockchain.CurrentSafeBlock() } else { block = api.eth.blockchain.GetBlockByNumber(uint64(blockNr)) } @@ -300,20 +287,8 @@ func (api *PublicDebugAPI) DumpBlock(blockNr rpc.BlockNumber) (state.Dump, error return stateDb.RawDump(opts), nil } -// PrivateDebugAPI is the collection of Ethereum full node APIs exposed over -// the private debugging endpoint. -type PrivateDebugAPI struct { - eth *Ethereum -} - -// NewPrivateDebugAPI creates a new API definition for the full node-related -// private debug methods of the Ethereum service. -func NewPrivateDebugAPI(eth *Ethereum) *PrivateDebugAPI { - return &PrivateDebugAPI{eth: eth} -} - // Preimage is a debug API function that returns the preimage for a sha3 hash, if known. -func (api *PrivateDebugAPI) Preimage(ctx context.Context, hash common.Hash) (hexutil.Bytes, error) { +func (api *DebugAPI) Preimage(ctx context.Context, hash common.Hash) (hexutil.Bytes, error) { if preimage := rawdb.ReadPreimage(api.eth.ChainDb(), hash); preimage != nil { return preimage, nil } @@ -328,8 +303,8 @@ type BadBlockArgs struct { } // GetBadBlocks returns a list of the last 'bad blocks' that the client has seen on the network -// and returns them as a JSON list of block-hashes -func (api *PrivateDebugAPI) GetBadBlocks(ctx context.Context) ([]*BadBlockArgs, error) { +// and returns them as a JSON list of block hashes. +func (api *DebugAPI) GetBadBlocks(ctx context.Context) ([]*BadBlockArgs, error) { var ( err error blocks = rawdb.ReadAllBadBlocks(api.eth.chainDb) @@ -343,7 +318,7 @@ func (api *PrivateDebugAPI) GetBadBlocks(ctx context.Context) ([]*BadBlockArgs, if rlpBytes, err := rlp.EncodeToBytes(block); err != nil { blockRlp = err.Error() // Hacky, but hey, it works } else { - blockRlp = fmt.Sprintf("0x%x", rlpBytes) + blockRlp = fmt.Sprintf("%#x", rlpBytes) } if blockJSON, err = ethapi.RPCMarshalBlock(block, true, true, api.eth.APIBackend.ChainConfig()); err != nil { blockJSON = map[string]interface{}{"error": err.Error()} @@ -361,7 +336,7 @@ func (api *PrivateDebugAPI) GetBadBlocks(ctx context.Context) ([]*BadBlockArgs, const AccountRangeMaxResults = 256 // AccountRange enumerates all accounts in the given block and start point in paging request -func (api *PublicDebugAPI) AccountRange(blockNrOrHash rpc.BlockNumberOrHash, start []byte, maxResults int, nocode, nostorage, incompletes bool) (state.IteratorDump, error) { +func (api *DebugAPI) AccountRange(blockNrOrHash rpc.BlockNumberOrHash, start hexutil.Bytes, maxResults int, nocode, nostorage, incompletes bool) (state.IteratorDump, error) { var stateDb *state.StateDB var err error @@ -377,6 +352,8 @@ func (api *PublicDebugAPI) AccountRange(blockNrOrHash rpc.BlockNumberOrHash, sta block = api.eth.blockchain.CurrentBlock() } else if number == rpc.FinalizedBlockNumber { block = api.eth.blockchain.CurrentFinalizedBlock() + } else if number == rpc.SafeBlockNumber { + block = api.eth.blockchain.CurrentSafeBlock() } else { block = api.eth.blockchain.GetBlockByNumber(uint64(number)) } @@ -428,7 +405,7 @@ type storageEntry struct { } // StorageRangeAt returns the storage at the given block height and transaction index. -func (api *PrivateDebugAPI) StorageRangeAt(blockHash common.Hash, txIndex int, contractAddress common.Address, keyStart hexutil.Bytes, maxResult int) (StorageRangeResult, error) { +func (api *DebugAPI) StorageRangeAt(blockHash common.Hash, txIndex int, contractAddress common.Address, keyStart hexutil.Bytes, maxResult int) (StorageRangeResult, error) { // Retrieve the block block := api.eth.blockchain.GetBlockByHash(blockHash) if block == nil { @@ -473,7 +450,7 @@ func storageRangeAt(st state.Trie, start []byte, maxResult int) (StorageRangeRes // code hash, or storage hash. // // With one parameter, returns the list of accounts modified in the specified block. -func (api *PrivateDebugAPI) GetModifiedAccountsByNumber(startNum uint64, endNum *uint64) ([]common.Address, error) { +func (api *DebugAPI) GetModifiedAccountsByNumber(startNum uint64, endNum *uint64) ([]common.Address, error) { var startBlock, endBlock *types.Block startBlock = api.eth.blockchain.GetBlockByNumber(startNum) @@ -501,7 +478,7 @@ func (api *PrivateDebugAPI) GetModifiedAccountsByNumber(startNum uint64, endNum // code hash, or storage hash. // // With one parameter, returns the list of accounts modified in the specified block. -func (api *PrivateDebugAPI) GetModifiedAccountsByHash(startHash common.Hash, endHash *common.Hash) ([]common.Address, error) { +func (api *DebugAPI) GetModifiedAccountsByHash(startHash common.Hash, endHash *common.Hash) ([]common.Address, error) { var startBlock, endBlock *types.Block startBlock = api.eth.blockchain.GetBlockByHash(startHash) if startBlock == nil { @@ -523,17 +500,17 @@ func (api *PrivateDebugAPI) GetModifiedAccountsByHash(startHash common.Hash, end return api.getModifiedAccounts(startBlock, endBlock) } -func (api *PrivateDebugAPI) getModifiedAccounts(startBlock, endBlock *types.Block) ([]common.Address, error) { +func (api *DebugAPI) getModifiedAccounts(startBlock, endBlock *types.Block) ([]common.Address, error) { if startBlock.Number().Uint64() >= endBlock.Number().Uint64() { return nil, fmt.Errorf("start block height (%d) must be less than end block height (%d)", startBlock.Number().Uint64(), endBlock.Number().Uint64()) } triedb := api.eth.BlockChain().StateCache().TrieDB() - oldTrie, err := trie.NewSecure(startBlock.Root(), triedb) + oldTrie, err := trie.NewStateTrie(common.Hash{}, startBlock.Root(), triedb) if err != nil { return nil, err } - newTrie, err := trie.NewSecure(endBlock.Root(), triedb) + newTrie, err := trie.NewStateTrie(common.Hash{}, endBlock.Root(), triedb) if err != nil { return nil, err } @@ -556,7 +533,7 @@ func (api *PrivateDebugAPI) getModifiedAccounts(startBlock, endBlock *types.Bloc // of the next block. // The (from, to) parameters are the sequence of blocks to search, which can go // either forwards or backwards -func (api *PrivateDebugAPI) GetAccessibleState(from, to rpc.BlockNumber) (uint64, error) { +func (api *DebugAPI) GetAccessibleState(from, to rpc.BlockNumber) (uint64, error) { db := api.eth.ChainDb() var pivot uint64 if p := rawdb.ReadLastPivotNumber(db); p != nil { @@ -609,5 +586,5 @@ func (api *PrivateDebugAPI) GetAccessibleState(from, to rpc.BlockNumber) (uint64 return uint64(i), nil } } - return 0, fmt.Errorf("No state found") + return 0, errors.New("no state found") } diff --git a/eth/api_backend.go b/eth/api_backend.go index f942710e2d8d6..00ecacc31df75 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -19,7 +19,6 @@ package eth import ( "context" "errors" - "fmt" "math/big" "time" @@ -74,7 +73,18 @@ func (b *EthAPIBackend) HeaderByNumber(ctx context.Context, number rpc.BlockNumb return b.eth.blockchain.CurrentBlock().Header(), nil } if number == rpc.FinalizedBlockNumber { - return b.eth.blockchain.CurrentFinalizedBlock().Header(), nil + block := b.eth.blockchain.CurrentFinalizedBlock() + if block != nil { + return block.Header(), nil + } + return nil, errors.New("finalized block not found") + } + if number == rpc.SafeBlockNumber { + block := b.eth.blockchain.CurrentSafeBlock() + if block != nil { + return block.Header(), nil + } + return nil, errors.New("safe block not found") } return b.eth.blockchain.GetHeaderByNumber(uint64(number)), nil } @@ -113,6 +123,9 @@ func (b *EthAPIBackend) BlockByNumber(ctx context.Context, number rpc.BlockNumbe if number == rpc.FinalizedBlockNumber { return b.eth.blockchain.CurrentFinalizedBlock(), nil } + if number == rpc.SafeBlockNumber { + return b.eth.blockchain.CurrentSafeBlock(), nil + } return b.eth.blockchain.GetBlockByNumber(uint64(number)), nil } @@ -188,17 +201,8 @@ func (b *EthAPIBackend) GetReceipts(ctx context.Context, hash common.Hash) (type return b.eth.blockchain.GetReceiptsByHash(hash), nil } -func (b *EthAPIBackend) GetLogs(ctx context.Context, hash common.Hash) ([][]*types.Log, error) { - db := b.eth.ChainDb() - number := rawdb.ReadHeaderNumber(db, hash) - if number == nil { - return nil, fmt.Errorf("failed to get block number for hash %#x", hash) - } - logs := rawdb.ReadLogs(db, hash, *number, b.eth.blockchain.Config()) - if logs == nil { - return nil, fmt.Errorf("failed to get logs for block #%d (0x%s)", *number, hash.TerminalString()) - } - return logs, nil +func (b *EthAPIBackend) GetLogs(ctx context.Context, hash common.Hash, number uint64) ([][]*types.Log, error) { + return rawdb.ReadLogs(b.eth.chainDb, hash, number, b.ChainConfig()), nil } func (b *EthAPIBackend) GetTd(ctx context.Context, hash common.Hash) *big.Int { diff --git a/eth/api_test.go b/eth/api_test.go index 39a1d5846004c..250591c1079b6 100644 --- a/eth/api_test.go +++ b/eth/api_test.go @@ -29,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/trie" ) var dumper = spew.ConfigState{Indent: " "} @@ -66,7 +67,7 @@ func TestAccountRange(t *testing.T) { t.Parallel() var ( - statedb = state.NewDatabaseWithConfig(rawdb.NewMemoryDatabase(), nil) + statedb = state.NewDatabaseWithConfig(rawdb.NewMemoryDatabase(), &trie.Config{Preimages: true}) state, _ = state.New(common.Hash{}, statedb, nil) addrs = [AccountRangeMaxResults * 2]common.Address{} m = map[common.Address]bool{} @@ -213,7 +214,7 @@ func TestStorageRangeAt(t *testing.T) { t.Error(err) } if !reflect.DeepEqual(result, test.want) { - t.Fatalf("wrong result for range 0x%x.., limit %d:\ngot %s\nwant %s", + t.Fatalf("wrong result for range %#x.., limit %d:\ngot %s\nwant %s", test.start, test.limit, dumper.Sdump(result), dumper.Sdump(&test.want)) } } diff --git a/eth/backend.go b/eth/backend.go index 8e70723b5b160..7782076363443 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -22,9 +22,9 @@ import ( "fmt" "math/big" "runtime" + "strings" "sync" "sync/atomic" - "time" "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/common" @@ -40,7 +40,6 @@ import ( "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/ethconfig" - "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/eth/gasprice" "github.com/ethereum/go-ethereum/eth/protocols/eth" "github.com/ethereum/go-ethereum/eth/protocols/snap" @@ -93,7 +92,7 @@ type Ethereum struct { etherbase common.Address networkID uint64 - netRPCService *ethapi.PublicNetAPI + netRPCService *ethapi.NetAPI p2pServer *p2p.Server @@ -136,11 +135,17 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { if err != nil { return nil, err } - chainConfig, genesisHash, genesisErr := core.SetupGenesisBlockWithOverride(chainDb, config.Genesis, config.OverrideArrowGlacier, config.OverrideTerminalTotalDifficulty) + chainConfig, genesisHash, genesisErr := core.SetupGenesisBlockWithOverride(chainDb, config.Genesis, config.OverrideTerminalTotalDifficulty, config.OverrideTerminalTotalDifficultyPassed) if _, ok := genesisErr.(*params.ConfigCompatError); genesisErr != nil && !ok { return nil, genesisErr } - log.Info("Initialised chain configuration", "config", chainConfig) + log.Info("") + log.Info(strings.Repeat("-", 153)) + for _, line := range strings.Split(chainConfig.String(), "\n") { + log.Info(line) + } + log.Info(strings.Repeat("-", 153)) + log.Info("") if err := pruner.RecoverPruning(stack.ResolvePath(""), chainDb, stack.ResolvePath(config.TrieCleanCacheJournal)); err != nil { log.Error("Failed to recover state", "error", err) @@ -259,7 +264,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { } // Start the RPC service - eth.netRPCService = ethapi.NewPublicNetAPI(eth.p2pServer, config.NetworkId) + eth.netRPCService = ethapi.NewNetAPI(eth.p2pServer, config.NetworkId) // Register the backend on the node stack.RegisterAPIs(eth.APIs()) @@ -301,47 +306,22 @@ func (s *Ethereum) APIs() []rpc.API { return append(apis, []rpc.API{ { Namespace: "eth", - Version: "1.0", - Service: NewPublicEthereumAPI(s), - Public: true, - }, { - Namespace: "eth", - Version: "1.0", - Service: NewPublicMinerAPI(s), - Public: true, - }, { - Namespace: "eth", - Version: "1.0", - Service: downloader.NewPublicDownloaderAPI(s.handler.downloader, s.eventMux), - Public: true, + Service: NewEthereumAPI(s), }, { Namespace: "miner", - Version: "1.0", - Service: NewPrivateMinerAPI(s), - Public: false, + Service: NewMinerAPI(s), }, { Namespace: "eth", - Version: "1.0", - Service: filters.NewPublicFilterAPI(s.APIBackend, false, 5*time.Minute), - Public: true, + Service: downloader.NewDownloaderAPI(s.handler.downloader, s.eventMux), }, { Namespace: "admin", - Version: "1.0", - Service: NewPrivateAdminAPI(s), - }, { - Namespace: "debug", - Version: "1.0", - Service: NewPublicDebugAPI(s), - Public: true, + Service: NewAdminAPI(s), }, { Namespace: "debug", - Version: "1.0", - Service: NewPrivateDebugAPI(s), + Service: NewDebugAPI(s), }, { Namespace: "net", - Version: "1.0", Service: s.netRPCService, - Public: true, }, }...) } diff --git a/eth/catalyst/api.go b/eth/catalyst/api.go index 108ec412d9e3f..b159f34e64ba4 100644 --- a/eth/catalyst/api.go +++ b/eth/catalyst/api.go @@ -22,6 +22,7 @@ import ( "encoding/binary" "errors" "fmt" + "math/big" "sync" "time" @@ -31,57 +32,118 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth" + "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/rpc" ) -// Register adds catalyst APIs to the full node. +// Register adds the engine API to the full node. func Register(stack *node.Node, backend *eth.Ethereum) error { - log.Warn("Catalyst mode enabled", "protocol", "eth") + log.Warn("Engine API enabled", "protocol", "eth") stack.RegisterAPIs([]rpc.API{ { Namespace: "engine", - Version: "1.0", Service: NewConsensusAPI(backend), - Public: true, Authenticated: true, }, - { - Namespace: "engine", - Version: "1.0", - Service: NewConsensusAPI(backend), - Public: true, - Authenticated: false, - }, }) return nil } +const ( + // invalidBlockHitEviction is the number of times an invalid block can be + // referenced in forkchoice update or new payload before it is attempted + // to be reprocessed again. + invalidBlockHitEviction = 128 + + // invalidTipsetsCap is the max number of recent block hashes tracked that + // have lead to some bad ancestor block. It's just an OOM protection. + invalidTipsetsCap = 512 + + // beaconUpdateStartupTimeout is the time to wait for a beacon client to get + // attached before starting to issue warnings. + beaconUpdateStartupTimeout = 30 * time.Second + + // beaconUpdateExchangeTimeout is the max time allowed for a beacon client to + // do a transition config exchange before it's considered offline and the user + // is warned. + beaconUpdateExchangeTimeout = 2 * time.Minute + + // beaconUpdateConsensusTimeout is the max time allowed for a beacon client + // to send a consensus update before it's considered offline and the user is + // warned. + beaconUpdateConsensusTimeout = 30 * time.Second + + // beaconUpdateWarnFrequency is the frequency at which to warn the user that + // the beacon client is offline. + beaconUpdateWarnFrequency = 5 * time.Minute +) + type ConsensusAPI struct { - eth *eth.Ethereum + eth *eth.Ethereum + remoteBlocks *headerQueue // Cache of remote payloads received localBlocks *payloadQueue // Cache of local payloads generated - // Lock for the forkChoiceUpdated method - forkChoiceLock sync.Mutex + + // The forkchoice update and new payload method require us to return the + // latest valid hash in an invalid chain. To support that return, we need + // to track historical bad blocks as well as bad tipsets in case a chain + // is constantly built on it. + // + // There are a few important caveats in this mechanism: + // - The bad block tracking is ephemeral, in-memory only. We must never + // persist any bad block information to disk as a bug in Geth could end + // up blocking a valid chain, even if a later Geth update would accept + // it. + // - Bad blocks will get forgotten after a certain threshold of import + // attempts and will be retried. The rationale is that if the network + // really-really-really tries to feed us a block, we should give it a + // new chance, perhaps us being racey instead of the block being legit + // bad (this happened in Geth at a point with import vs. pending race). + // - Tracking all the blocks built on top of the bad one could be a bit + // problematic, so we will only track the head chain segment of a bad + // chain to allow discarding progressing bad chains and side chains, + // without tracking too much bad data. + invalidBlocksHits map[common.Hash]int // Emhemeral cache to track invalid blocks and their hit count + invalidTipsets map[common.Hash]*types.Header // Ephemeral cache to track invalid tipsets and their bad ancestor + invalidLock sync.Mutex // Protects the invalid maps from concurrent access + + // Geth can appear to be stuck or do strange things if the beacon client is + // offline or is sending us strange data. Stash some update stats away so + // that we can warn the user and not have them open issues on our tracker. + lastTransitionUpdate time.Time + lastTransitionLock sync.Mutex + lastForkchoiceUpdate time.Time + lastForkchoiceLock sync.Mutex + lastNewPayloadUpdate time.Time + lastNewPayloadLock sync.Mutex + + forkchoiceLock sync.Mutex // Lock for the forkChoiceUpdated method } // NewConsensusAPI creates a new consensus api for the given backend. // The underlying blockchain needs to have a valid terminal total difficulty set. func NewConsensusAPI(eth *eth.Ethereum) *ConsensusAPI { if eth.BlockChain().Config().TerminalTotalDifficulty == nil { - panic("Catalyst started without valid total difficulty") + log.Warn("Engine API started but chain not configured for merge yet") } - return &ConsensusAPI{ - eth: eth, - remoteBlocks: newHeaderQueue(), - localBlocks: newPayloadQueue(), + api := &ConsensusAPI{ + eth: eth, + remoteBlocks: newHeaderQueue(), + localBlocks: newPayloadQueue(), + invalidBlocksHits: make(map[common.Hash]int), + invalidTipsets: make(map[common.Hash]*types.Header), } + eth.Downloader().SetBadBlockCallback(api.setInvalidAncestor) + go api.heartbeat() + + return api } // ForkchoiceUpdatedV1 has several responsibilities: // If the method is called with an empty head block: -// we return success, which can be used to check if the catalyst mode is enabled +// we return success, which can be used to check if the engine API is enabled // If the total difficulty was not reached: // we return INVALID // If the finalizedBlockHash is set: @@ -90,20 +152,28 @@ func NewConsensusAPI(eth *eth.Ethereum) *ConsensusAPI { // If there are payloadAttributes: // we try to assemble a block with the payloadAttributes and return its payloadID func (api *ConsensusAPI) ForkchoiceUpdatedV1(update beacon.ForkchoiceStateV1, payloadAttributes *beacon.PayloadAttributesV1) (beacon.ForkChoiceResponse, error) { - api.forkChoiceLock.Lock() - defer api.forkChoiceLock.Unlock() + api.forkchoiceLock.Lock() + defer api.forkchoiceLock.Unlock() log.Trace("Engine API request received", "method", "ForkchoiceUpdated", "head", update.HeadBlockHash, "finalized", update.FinalizedBlockHash, "safe", update.SafeBlockHash) if update.HeadBlockHash == (common.Hash{}) { log.Warn("Forkchoice requested update to zero hash") return beacon.STATUS_INVALID, nil // TODO(karalabe): Why does someone send us this? } + // Stash away the last update to warn the user if the beacon client goes offline + api.lastForkchoiceLock.Lock() + api.lastForkchoiceUpdate = time.Now() + api.lastForkchoiceLock.Unlock() // Check whether we have the block yet in our database or not. If not, we'll // need to either trigger a sync, or to reject this forkchoice update for a // reason. block := api.eth.BlockChain().GetBlockByHash(update.HeadBlockHash) if block == nil { + // If this block was previously invalidated, keep rejecting it here too + if res := api.checkInvalidAncestor(update.HeadBlockHash, update.HeadBlockHash); res != nil { + return beacon.ForkChoiceResponse{PayloadStatus: *res, PayloadID: nil}, nil + } // If the head hash is unknown (was not given to us in a newPayload request), // we cannot resolve the header, so not much to do. This could be extended in // the future to resolve from the `eth` network, but it's an unexpected case @@ -138,21 +208,35 @@ func (api *ConsensusAPI) ForkchoiceUpdatedV1(update beacon.ForkchoiceStateV1, pa log.Error("TDs unavailable for TTD check", "number", block.NumberU64(), "hash", update.HeadBlockHash, "td", td, "parent", block.ParentHash(), "ptd", ptd) return beacon.STATUS_INVALID, errors.New("TDs unavailable for TDD check") } - if td.Cmp(ttd) < 0 || (block.NumberU64() > 0 && ptd.Cmp(ttd) > 0) { + if td.Cmp(ttd) < 0 { log.Error("Refusing beacon update to pre-merge", "number", block.NumberU64(), "hash", update.HeadBlockHash, "diff", block.Difficulty(), "age", common.PrettyAge(time.Unix(int64(block.Time()), 0))) return beacon.ForkChoiceResponse{PayloadStatus: beacon.INVALID_TERMINAL_BLOCK, PayloadID: nil}, nil } + if block.NumberU64() > 0 && ptd.Cmp(ttd) >= 0 { + log.Error("Parent block is already post-ttd", "number", block.NumberU64(), "hash", update.HeadBlockHash, "diff", block.Difficulty(), "age", common.PrettyAge(time.Unix(int64(block.Time()), 0))) + return beacon.ForkChoiceResponse{PayloadStatus: beacon.INVALID_TERMINAL_BLOCK, PayloadID: nil}, nil + } + } + valid := func(id *beacon.PayloadID) beacon.ForkChoiceResponse { + return beacon.ForkChoiceResponse{ + PayloadStatus: beacon.PayloadStatusV1{Status: beacon.VALID, LatestValidHash: &update.HeadBlockHash}, + PayloadID: id, + } } - if rawdb.ReadCanonicalHash(api.eth.ChainDb(), block.NumberU64()) != update.HeadBlockHash { // Block is not canonical, set head. if latestValid, err := api.eth.BlockChain().SetCanonical(block); err != nil { return beacon.ForkChoiceResponse{PayloadStatus: beacon.PayloadStatusV1{Status: beacon.INVALID, LatestValidHash: &latestValid}}, err } + } else if api.eth.BlockChain().CurrentBlock().Hash() == update.HeadBlockHash { + // If the specified head matches with our local head, do nothing and keep + // generating the payload. It's a special corner case that a few slots are + // missing and we are requested to generate the payload in slot. } else { // If the head block is already in our canonical chain, the beacon client is // probably resyncing. Ignore the update. log.Info("Ignoring beacon update to old head", "number", block.NumberU64(), "hash", update.HeadBlockHash, "age", common.PrettyAge(time.Unix(int64(block.Time()), 0)), "have", api.eth.BlockChain().CurrentBlock().NumberU64()) + return valid(nil), nil } api.eth.SetSynced() @@ -185,12 +269,8 @@ func (api *ConsensusAPI) ForkchoiceUpdatedV1(update beacon.ForkchoiceStateV1, pa log.Warn("Safe block not in canonical chain") return beacon.STATUS_INVALID, beacon.InvalidForkChoiceState.With(errors.New("safe block not in canonical chain")) } - } - valid := func(id *beacon.PayloadID) beacon.ForkChoiceResponse { - return beacon.ForkChoiceResponse{ - PayloadStatus: beacon.PayloadStatusV1{Status: beacon.VALID, LatestValidHash: &update.HeadBlockHash}, - PayloadID: id, - } + // Set the safe block + api.eth.BlockChain().SetSafe(safeBlock) } // If payload generation was requested, create a new block to be potentially // sealed by the beacon client. The payload will be requested later, and we @@ -219,15 +299,20 @@ func (api *ConsensusAPI) ForkchoiceUpdatedV1(update beacon.ForkchoiceStateV1, pa // ExchangeTransitionConfigurationV1 checks the given configuration against // the configuration of the node. func (api *ConsensusAPI) ExchangeTransitionConfigurationV1(config beacon.TransitionConfigurationV1) (*beacon.TransitionConfigurationV1, error) { + log.Trace("Engine API request received", "method", "ExchangeTransitionConfiguration", "ttd", config.TerminalTotalDifficulty) if config.TerminalTotalDifficulty == nil { return nil, errors.New("invalid terminal total difficulty") } + // Stash away the last update to warn the user if the beacon client goes offline + api.lastTransitionLock.Lock() + api.lastTransitionUpdate = time.Now() + api.lastTransitionLock.Unlock() + ttd := api.eth.BlockChain().Config().TerminalTotalDifficulty - if ttd.Cmp(config.TerminalTotalDifficulty.ToInt()) != 0 { + if ttd == nil || ttd.Cmp(config.TerminalTotalDifficulty.ToInt()) != 0 { log.Warn("Invalid TTD configured", "geth", ttd, "beacon", config.TerminalTotalDifficulty) return nil, fmt.Errorf("invalid ttd: execution %v consensus %v", ttd, config.TerminalTotalDifficulty) } - if config.TerminalBlockHash != (common.Hash{}) { if hash := api.eth.BlockChain().GetCanonicalHash(uint64(config.TerminalBlockNumber)); hash == config.TerminalBlockHash { return &beacon.TransitionConfigurationV1{ @@ -259,6 +344,11 @@ func (api *ConsensusAPI) NewPayloadV1(params beacon.ExecutableDataV1) (beacon.Pa log.Debug("Invalid NewPayload params", "params", params, "error", err) return beacon.PayloadStatusV1{Status: beacon.INVALIDBLOCKHASH}, nil } + // Stash away the last update to warn the user if the beacon client goes offline + api.lastNewPayloadLock.Lock() + api.lastNewPayloadUpdate = time.Now() + api.lastNewPayloadLock.Unlock() + // If we already have the block locally, ignore the entire execution and just // return a fake success. if block := api.eth.BlockChain().GetBlockByHash(params.BlockHash); block != nil { @@ -266,6 +356,10 @@ func (api *ConsensusAPI) NewPayloadV1(params beacon.ExecutableDataV1) (beacon.Pa hash := block.Hash() return beacon.PayloadStatusV1{Status: beacon.VALID, LatestValidHash: &hash}, nil } + // If this block was rejected previously, keep rejecting it + if res := api.checkInvalidAncestor(block.Hash(), block.Hash()); res != nil { + return *res, nil + } // If the parent is missing, we - in theory - could trigger a sync, but that // would also entail a reorg. That is problematic if multiple sibling blocks // are being fed to us, and even more so, if some semi-distant uncle shortens @@ -274,37 +368,33 @@ func (api *ConsensusAPI) NewPayloadV1(params beacon.ExecutableDataV1) (beacon.Pa // update after legit payload executions. parent := api.eth.BlockChain().GetBlock(block.ParentHash(), block.NumberU64()-1) if parent == nil { - // Stash the block away for a potential forced forckchoice update to it - // at a later time. - api.remoteBlocks.put(block.Hash(), block.Header()) - - // Although we don't want to trigger a sync, if there is one already in - // progress, try to extend if with the current payload request to relieve - // some strain from the forkchoice update. - if err := api.eth.Downloader().BeaconExtend(api.eth.SyncMode(), block.Header()); err == nil { - log.Debug("Payload accepted for sync extension", "number", params.Number, "hash", params.BlockHash) - return beacon.PayloadStatusV1{Status: beacon.SYNCING}, nil - } - // Either no beacon sync was started yet, or it rejected the delivered - // payload as non-integratable on top of the existing sync. We'll just - // have to rely on the beacon client to forcefully update the head with - // a forkchoice update request. - log.Warn("Ignoring payload with missing parent", "number", params.Number, "hash", params.BlockHash, "parent", params.ParentHash) - return beacon.PayloadStatusV1{Status: beacon.ACCEPTED}, nil + return api.delayPayloadImport(block) } // We have an existing parent, do some sanity checks to avoid the beacon client // triggering too early var ( - td = api.eth.BlockChain().GetTd(parent.Hash(), parent.NumberU64()) - ttd = api.eth.BlockChain().Config().TerminalTotalDifficulty + ptd = api.eth.BlockChain().GetTd(parent.Hash(), parent.NumberU64()) + ttd = api.eth.BlockChain().Config().TerminalTotalDifficulty + gptd = api.eth.BlockChain().GetTd(parent.ParentHash(), parent.NumberU64()-1) ) - if td.Cmp(ttd) < 0 { - log.Warn("Ignoring pre-merge payload", "number", params.Number, "hash", params.BlockHash, "td", td, "ttd", ttd) + if ptd.Cmp(ttd) < 0 { + log.Warn("Ignoring pre-merge payload", "number", params.Number, "hash", params.BlockHash, "td", ptd, "ttd", ttd) + return beacon.INVALID_TERMINAL_BLOCK, nil + } + if parent.Difficulty().BitLen() > 0 && gptd != nil && gptd.Cmp(ttd) >= 0 { + log.Error("Ignoring pre-merge parent block", "number", params.Number, "hash", params.BlockHash, "td", ptd, "ttd", ttd) return beacon.INVALID_TERMINAL_BLOCK, nil } if block.Time() <= parent.Time() { log.Warn("Invalid timestamp", "parent", block.Time(), "block", block.Time()) - return api.invalid(errors.New("invalid timestamp"), parent), nil + return api.invalid(errors.New("invalid timestamp"), parent.Header()), nil + } + // Another cornercase: if the node is in snap sync mode, but the CL client + // tries to make it import a block. That should be denied as pushing something + // into the database directly will conflict with the assumptions of snap sync + // that it has an empty db that it can fill itself. + if api.eth.SyncMode() != downloader.FullSync { + return api.delayPayloadImport(block) } if !api.eth.BlockChain().HasBlockAndState(block.ParentHash(), block.NumberU64()-1) { api.remoteBlocks.put(block.Hash(), block.Header()) @@ -314,7 +404,13 @@ func (api *ConsensusAPI) NewPayloadV1(params beacon.ExecutableDataV1) (beacon.Pa log.Trace("Inserting block without sethead", "hash", block.Hash(), "number", block.Number) if err := api.eth.BlockChain().InsertBlockWithoutSetHead(block); err != nil { log.Warn("NewPayloadV1: inserting block failed", "error", err) - return api.invalid(err, parent), nil + + api.invalidLock.Lock() + api.invalidBlocksHits[block.Hash()] = 1 + api.invalidTipsets[block.Hash()] = block.Header() + api.invalidLock.Unlock() + + return api.invalid(err, parent.Header()), nil } // We've accepted a valid payload from the beacon client. Mark the local // chain transitions to notify other subsystems (e.g. downloader) of the @@ -340,13 +436,242 @@ func computePayloadId(headBlockHash common.Hash, params *beacon.PayloadAttribute return out } +// delayPayloadImport stashes the given block away for import at a later time, +// either via a forkchoice update or a sync extension. This method is meant to +// be called by the newpayload command when the block seems to be ok, but some +// prerequisite prevents it from being processed (e.g. no parent, or snap sync). +func (api *ConsensusAPI) delayPayloadImport(block *types.Block) (beacon.PayloadStatusV1, error) { + // Sanity check that this block's parent is not on a previously invalidated + // chain. If it is, mark the block as invalid too. + if res := api.checkInvalidAncestor(block.ParentHash(), block.Hash()); res != nil { + return *res, nil + } + // Stash the block away for a potential forced forkchoice update to it + // at a later time. + api.remoteBlocks.put(block.Hash(), block.Header()) + + // Although we don't want to trigger a sync, if there is one already in + // progress, try to extend if with the current payload request to relieve + // some strain from the forkchoice update. + if err := api.eth.Downloader().BeaconExtend(api.eth.SyncMode(), block.Header()); err == nil { + log.Debug("Payload accepted for sync extension", "number", block.NumberU64(), "hash", block.Hash()) + return beacon.PayloadStatusV1{Status: beacon.SYNCING}, nil + } + // Either no beacon sync was started yet, or it rejected the delivered + // payload as non-integratable on top of the existing sync. We'll just + // have to rely on the beacon client to forcefully update the head with + // a forkchoice update request. + if api.eth.SyncMode() == downloader.FullSync { + // In full sync mode, failure to import a well-formed block can only mean + // that the parent state is missing and the syncer rejected extending the + // current cycle with the new payload. + log.Warn("Ignoring payload with missing parent", "number", block.NumberU64(), "hash", block.Hash(), "parent", block.ParentHash()) + } else { + // In non-full sync mode (i.e. snap sync) all payloads are rejected until + // snap sync terminates as snap sync relies on direct database injections + // and cannot afford concurrent out-if-band modifications via imports. + log.Warn("Ignoring payload while snap syncing", "number", block.NumberU64(), "hash", block.Hash()) + } + return beacon.PayloadStatusV1{Status: beacon.SYNCING}, nil +} + +// setInvalidAncestor is a callback for the downloader to notify us if a bad block +// is encountered during the async sync. +func (api *ConsensusAPI) setInvalidAncestor(invalid *types.Header, origin *types.Header) { + api.invalidLock.Lock() + defer api.invalidLock.Unlock() + + api.invalidTipsets[origin.Hash()] = invalid + api.invalidBlocksHits[invalid.Hash()]++ +} + +// checkInvalidAncestor checks whether the specified chain end links to a known +// bad ancestor. If yes, it constructs the payload failure response to return. +func (api *ConsensusAPI) checkInvalidAncestor(check common.Hash, head common.Hash) *beacon.PayloadStatusV1 { + api.invalidLock.Lock() + defer api.invalidLock.Unlock() + + // If the hash to check is unknown, return valid + invalid, ok := api.invalidTipsets[check] + if !ok { + return nil + } + // If the bad hash was hit too many times, evict it and try to reprocess in + // the hopes that we have a data race that we can exit out of. + badHash := invalid.Hash() + + api.invalidBlocksHits[badHash]++ + if api.invalidBlocksHits[badHash] >= invalidBlockHitEviction { + log.Warn("Too many bad block import attempt, trying", "number", invalid.Number, "hash", badHash) + delete(api.invalidBlocksHits, badHash) + + for descendant, badHeader := range api.invalidTipsets { + if badHeader.Hash() == badHash { + delete(api.invalidTipsets, descendant) + } + } + return nil + } + // Not too many failures yet, mark the head of the invalid chain as invalid + if check != head { + log.Warn("Marked new chain head as invalid", "hash", head, "badnumber", invalid.Number, "badhash", badHash) + for len(api.invalidTipsets) >= invalidTipsetsCap { + for key := range api.invalidTipsets { + delete(api.invalidTipsets, key) + break + } + } + api.invalidTipsets[head] = invalid + } + // If the last valid hash is the terminal pow block, return 0x0 for latest valid hash + lastValid := &invalid.ParentHash + if header := api.eth.BlockChain().GetHeader(invalid.ParentHash, invalid.Number.Uint64()-1); header != nil && header.Difficulty.Sign() != 0 { + lastValid = &common.Hash{} + } + failure := "links to previously rejected block" + return &beacon.PayloadStatusV1{ + Status: beacon.INVALID, + LatestValidHash: lastValid, + ValidationError: &failure, + } +} + // invalid returns a response "INVALID" with the latest valid hash supplied by latest or to the current head // if no latestValid block was provided. -func (api *ConsensusAPI) invalid(err error, latestValid *types.Block) beacon.PayloadStatusV1 { +func (api *ConsensusAPI) invalid(err error, latestValid *types.Header) beacon.PayloadStatusV1 { currentHash := api.eth.BlockChain().CurrentBlock().Hash() if latestValid != nil { - currentHash = latestValid.Hash() + // Set latest valid hash to 0x0 if parent is PoW block + currentHash = common.Hash{} + if latestValid.Difficulty.BitLen() == 0 { + // Otherwise set latest valid hash to parent hash + currentHash = latestValid.Hash() + } } errorMsg := err.Error() return beacon.PayloadStatusV1{Status: beacon.INVALID, LatestValidHash: ¤tHash, ValidationError: &errorMsg} } + +// heartbeat loops indefinitely, and checks if there have been beacon client updates +// received in the last while. If not - or if they but strange ones - it warns the +// user that something might be off with their consensus node. +// +// TODO(karalabe): Spin this goroutine down somehow +func (api *ConsensusAPI) heartbeat() { + // Sleep a bit on startup since there's obviously no beacon client yet + // attached, so no need to print scary warnings to the user. + time.Sleep(beaconUpdateStartupTimeout) + + var ( + offlineLogged time.Time + ) + for { + // Sleep a bit and retrieve the last known consensus updates + time.Sleep(5 * time.Second) + + // If the network is not yet merged/merging, don't bother scaring the user + ttd := api.eth.BlockChain().Config().TerminalTotalDifficulty + if ttd == nil { + continue + } + api.lastTransitionLock.Lock() + lastTransitionUpdate := api.lastTransitionUpdate + api.lastTransitionLock.Unlock() + + api.lastForkchoiceLock.Lock() + lastForkchoiceUpdate := api.lastForkchoiceUpdate + api.lastForkchoiceLock.Unlock() + + api.lastNewPayloadLock.Lock() + lastNewPayloadUpdate := api.lastNewPayloadUpdate + api.lastNewPayloadLock.Unlock() + + // If there have been no updates for the past while, warn the user + // that the beacon client is probably offline + if api.eth.BlockChain().Config().TerminalTotalDifficultyPassed || api.eth.Merger().TDDReached() { + if time.Since(lastForkchoiceUpdate) > beaconUpdateConsensusTimeout && time.Since(lastNewPayloadUpdate) > beaconUpdateConsensusTimeout { + if time.Since(lastTransitionUpdate) > beaconUpdateExchangeTimeout { + if time.Since(offlineLogged) > beaconUpdateWarnFrequency { + if lastTransitionUpdate.IsZero() { + log.Warn("Post-merge network, but no beacon client seen. Please launch one to follow the chain!") + } else { + log.Warn("Previously seen beacon client is offline. Please ensure it is operational to follow the chain!") + } + offlineLogged = time.Now() + } + continue + } + if time.Since(offlineLogged) > beaconUpdateWarnFrequency { + if lastForkchoiceUpdate.IsZero() && lastNewPayloadUpdate.IsZero() { + log.Warn("Beacon client online, but never received consensus updates. Please ensure your beacon client is operational to follow the chain!") + } else { + log.Warn("Beacon client online, but no consensus updates received in a while. Please fix your beacon client to follow the chain!") + } + offlineLogged = time.Now() + } + continue + } else { + offlineLogged = time.Time{} + } + } else { + if time.Since(lastTransitionUpdate) > beaconUpdateExchangeTimeout { + if time.Since(offlineLogged) > beaconUpdateWarnFrequency { + // Retrieve the last few blocks and make a rough estimate as + // to when the merge transition should happen + var ( + chain = api.eth.BlockChain() + head = chain.CurrentBlock() + htd = chain.GetTd(head.Hash(), head.NumberU64()) + eta time.Duration + ) + if head.NumberU64() > 0 && htd.Cmp(ttd) < 0 { + // Accumulate the last 64 difficulties to estimate the growth + var diff float64 + + block := head + for i := 0; i < 64; i++ { + diff += float64(block.Difficulty().Uint64()) + if parent := chain.GetBlock(block.ParentHash(), block.NumberU64()-1); parent == nil { + break + } else { + block = parent + } + } + // Estimate an ETA based on the block times and the difficulty growth + growth := diff / float64(head.Time()-block.Time()+1) // +1 to avoid div by zero + if growth > 0 { + if left := new(big.Int).Sub(ttd, htd); left.IsUint64() { + eta = time.Duration(float64(left.Uint64())/growth) * time.Second + } else { + eta = time.Duration(new(big.Int).Div(left, big.NewInt(int64(growth))).Uint64()) * time.Second + } + } + } + var message string + if htd.Cmp(ttd) > 0 { + if lastTransitionUpdate.IsZero() { + message = "Merge already reached, but no beacon client seen. Please launch one to follow the chain!" + } else { + message = "Merge already reached, but previously seen beacon client is offline. Please ensure it is operational to follow the chain!" + } + } else { + if lastTransitionUpdate.IsZero() { + message = "Merge is configured, but no beacon client seen. Please ensure you have one available before the transition arrives!" + } else { + message = "Merge is configured, but previously seen beacon client is offline. Please ensure it is operational before the transition arrives!" + } + } + if eta == 0 { + log.Warn(message) + } else { + log.Warn(message, "eta", common.PrettyAge(time.Now().Add(-eta))) // weird hack, but duration formatted doesn't handle days + } + offlineLogged = time.Now() + } + continue + } else { + offlineLogged = time.Time{} + } + } + } +} diff --git a/eth/catalyst/api_test.go b/eth/catalyst/api_test.go index 415506d58e507..0d945993eb37a 100644 --- a/eth/catalyst/api_test.go +++ b/eth/catalyst/api_test.go @@ -69,7 +69,7 @@ func generatePreMergeChain(n int) (*core.Genesis, []*types.Block) { g.AddTx(tx) testNonce++ } - gblock := genesis.ToBlock(db) + gblock := genesis.MustCommit(db) engine := ethash.NewFaker() blocks, _ := core.GenerateChain(config, gblock, engine, db, n, generate) totalDifficulty := big.NewInt(0) @@ -205,7 +205,6 @@ func checkLogEvents(t *testing.T, logsCh <-chan []*types.Log, rmLogsCh <-chan co func TestInvalidPayloadTimestamp(t *testing.T) { genesis, preMergeBlocks := generatePreMergeChain(10) n, ethservice := startEthService(t, genesis, preMergeBlocks) - ethservice.Merger().ReachTTD() defer n.Close() var ( api = NewConsensusAPI(ethservice) @@ -250,7 +249,6 @@ func TestInvalidPayloadTimestamp(t *testing.T) { func TestEth2NewBlock(t *testing.T) { genesis, preMergeBlocks := generatePreMergeChain(10) n, ethservice := startEthService(t, genesis, preMergeBlocks) - ethservice.Merger().ReachTTD() defer n.Close() var ( @@ -405,7 +403,7 @@ func startEthService(t *testing.T, genesis *core.Genesis, blocks []*types.Block) t.Fatal("can't create node:", err) } - ethcfg := ðconfig.Config{Genesis: genesis, Ethash: ethash.Config{PowMode: ethash.ModeFake}, SyncMode: downloader.SnapSync, TrieTimeout: time.Minute, TrieDirtyCache: 256, TrieCleanCache: 256} + ethcfg := ðconfig.Config{Genesis: genesis, Ethash: ethash.Config{PowMode: ethash.ModeFake}, SyncMode: downloader.FullSync, TrieTimeout: time.Minute, TrieDirtyCache: 256, TrieCleanCache: 256} ethservice, err := eth.New(n, ethcfg) if err != nil { t.Fatal("can't create eth service:", err) @@ -427,7 +425,6 @@ func startEthService(t *testing.T, genesis *core.Genesis, blocks []*types.Block) func TestFullAPI(t *testing.T) { genesis, preMergeBlocks := generatePreMergeChain(10) n, ethservice := startEthService(t, genesis, preMergeBlocks) - ethservice.Merger().ReachTTD() defer n.Close() var ( parent = ethservice.BlockChain().CurrentBlock() @@ -480,7 +477,6 @@ func setupBlocks(t *testing.T, ethservice *eth.Ethereum, n int, parent *types.Bl func TestExchangeTransitionConfig(t *testing.T) { genesis, preMergeBlocks := generatePreMergeChain(10) n, ethservice := startEthService(t, genesis, preMergeBlocks) - ethservice.Merger().ReachTTD() defer n.Close() var ( api = NewConsensusAPI(ethservice) @@ -543,7 +539,6 @@ CommonAncestor◄─▲── P1 ◄── P2 ◄─ P3 ◄─ ... ◄─ Pn func TestNewPayloadOnInvalidChain(t *testing.T) { genesis, preMergeBlocks := generatePreMergeChain(10) n, ethservice := startEthService(t, genesis, preMergeBlocks) - ethservice.Merger().ReachTTD() defer n.Close() var ( @@ -618,7 +613,6 @@ func assembleBlock(api *ConsensusAPI, parentHash common.Hash, params *beacon.Pay func TestEmptyBlocks(t *testing.T) { genesis, preMergeBlocks := generatePreMergeChain(10) n, ethservice := startEthService(t, genesis, preMergeBlocks) - ethservice.Merger().ReachTTD() defer n.Close() commonAncestor := ethservice.BlockChain().CurrentBlock() @@ -653,7 +647,8 @@ func TestEmptyBlocks(t *testing.T) { if status.Status != beacon.INVALID { t.Errorf("invalid status: expected INVALID got: %v", status.Status) } - expected := commonAncestor.Hash() + // Expect 0x0 on INVALID block on top of PoW block + expected := common.Hash{} if !bytes.Equal(status.LatestValidHash[:], expected[:]) { t.Fatalf("invalid LVH: got %v want %v", status.LatestValidHash, expected) } @@ -667,8 +662,8 @@ func TestEmptyBlocks(t *testing.T) { if err != nil { t.Fatal(err) } - if status.Status != beacon.ACCEPTED { - t.Errorf("invalid status: expected ACCEPTED got: %v", status.Status) + if status.Status != beacon.SYNCING { + t.Errorf("invalid status: expected SYNCING got: %v", status.Status) } if status.LatestValidHash != nil { t.Fatalf("invalid LVH: got %v wanted nil", status.LatestValidHash) @@ -734,8 +729,6 @@ func TestTrickRemoteBlockCache(t *testing.T) { genesis, preMergeBlocks := generatePreMergeChain(10) nodeA, ethserviceA := startEthService(t, genesis, preMergeBlocks) nodeB, ethserviceB := startEthService(t, genesis, preMergeBlocks) - ethserviceA.Merger().ReachTTD() - ethserviceB.Merger().ReachTTD() defer nodeA.Close() defer nodeB.Close() for nodeB.Server().NodeInfo().Ports.Listener == 0 { @@ -780,8 +773,8 @@ func TestTrickRemoteBlockCache(t *testing.T) { if err != nil { panic(err) } - if status.Status == beacon.INVALID { - panic("success") + if status.Status == beacon.VALID { + t.Error("invalid status: VALID on an invalid chain") } // Now reorg to the head of the invalid chain resp, err := apiB.ForkchoiceUpdatedV1(beacon.ForkchoiceStateV1{HeadBlockHash: payload.BlockHash, SafeBlockHash: payload.BlockHash, FinalizedBlockHash: payload.ParentHash}, nil) @@ -789,8 +782,80 @@ func TestTrickRemoteBlockCache(t *testing.T) { t.Fatal(err) } if resp.PayloadStatus.Status == beacon.VALID { - t.Errorf("invalid status: expected INVALID got: %v", resp.PayloadStatus.Status) + t.Error("invalid status: VALID on an invalid chain") } time.Sleep(100 * time.Millisecond) } } + +func TestInvalidBloom(t *testing.T) { + genesis, preMergeBlocks := generatePreMergeChain(10) + n, ethservice := startEthService(t, genesis, preMergeBlocks) + ethservice.Merger().ReachTTD() + defer n.Close() + + commonAncestor := ethservice.BlockChain().CurrentBlock() + api := NewConsensusAPI(ethservice) + + // Setup 10 blocks on the canonical chain + setupBlocks(t, ethservice, 10, commonAncestor, func(parent *types.Block) {}) + + // (1) check LatestValidHash by sending a normal payload (P1'') + payload := getNewPayload(t, api, commonAncestor) + payload.LogsBloom = append(payload.LogsBloom, byte(1)) + status, err := api.NewPayloadV1(*payload) + if err != nil { + t.Fatal(err) + } + if status.Status != beacon.INVALIDBLOCKHASH { + t.Errorf("invalid status: expected VALID got: %v", status.Status) + } +} + +func TestNewPayloadOnInvalidTerminalBlock(t *testing.T) { + genesis, preMergeBlocks := generatePreMergeChain(100) + fmt.Println(genesis.Config.TerminalTotalDifficulty) + genesis.Config.TerminalTotalDifficulty = preMergeBlocks[0].Difficulty() //.Sub(genesis.Config.TerminalTotalDifficulty, preMergeBlocks[len(preMergeBlocks)-1].Difficulty()) + + fmt.Println(genesis.Config.TerminalTotalDifficulty) + n, ethservice := startEthService(t, genesis, preMergeBlocks) + defer n.Close() + + var ( + api = NewConsensusAPI(ethservice) + parent = preMergeBlocks[len(preMergeBlocks)-1] + ) + + // Test parent already post TTD in FCU + fcState := beacon.ForkchoiceStateV1{ + HeadBlockHash: parent.Hash(), + SafeBlockHash: common.Hash{}, + FinalizedBlockHash: common.Hash{}, + } + resp, err := api.ForkchoiceUpdatedV1(fcState, nil) + if err != nil { + t.Fatalf("error sending forkchoice, err=%v", err) + } + if resp.PayloadStatus != beacon.INVALID_TERMINAL_BLOCK { + t.Fatalf("error sending invalid forkchoice, invalid status: %v", resp.PayloadStatus.Status) + } + + // Test parent already post TTD in NewPayload + params := beacon.PayloadAttributesV1{ + Timestamp: parent.Time() + 1, + Random: crypto.Keccak256Hash([]byte{byte(1)}), + SuggestedFeeRecipient: parent.Coinbase(), + } + empty, err := api.eth.Miner().GetSealingBlockSync(parent.Hash(), params.Timestamp, params.SuggestedFeeRecipient, params.Random, true) + if err != nil { + t.Fatalf("error preparing payload, err=%v", err) + } + data := *beacon.BlockToExecutableData(empty) + resp2, err := api.NewPayloadV1(data) + if err != nil { + t.Fatalf("error sending NewPayload, err=%v", err) + } + if resp2 != beacon.INVALID_TERMINAL_BLOCK { + t.Fatalf("error sending invalid forkchoice, invalid status: %v", resp.PayloadStatus.Status) + } +} diff --git a/eth/discovery.go b/eth/discovery.go deleted file mode 100644 index f7c85b4c5d3ba..0000000000000 --- a/eth/discovery.go +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright 2020 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package eth - -import ( - "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/forkid" - "github.com/ethereum/go-ethereum/p2p/enode" - "github.com/ethereum/go-ethereum/rlp" -) - -// ethEntry is the "eth" ENR entry which advertises eth protocol -// on the discovery network. -type ethEntry struct { - ForkID forkid.ID // Fork identifier per EIP-2124 - - // Ignore additional fields (for forward compatibility). - Rest []rlp.RawValue `rlp:"tail"` -} - -// ENRKey implements enr.Entry. -func (e ethEntry) ENRKey() string { - return "eth" -} - -// startEthEntryUpdate starts the ENR updater loop. -func (eth *Ethereum) startEthEntryUpdate(ln *enode.LocalNode) { - var newHead = make(chan core.ChainHeadEvent, 10) - sub := eth.blockchain.SubscribeChainHeadEvent(newHead) - - go func() { - defer sub.Unsubscribe() - for { - select { - case <-newHead: - ln.Set(eth.currentEthEntry()) - case <-sub.Err(): - // Would be nice to sync with eth.Stop, but there is no - // good way to do that. - return - } - } - }() -} - -func (eth *Ethereum) currentEthEntry() *ethEntry { - return ðEntry{ForkID: forkid.NewID(eth.blockchain.Config(), eth.blockchain.Genesis().Hash(), - eth.blockchain.CurrentHeader().Number.Uint64())} -} diff --git a/eth/downloader/api.go b/eth/downloader/api.go index 2024d23deade6..b3f7113bcde9e 100644 --- a/eth/downloader/api.go +++ b/eth/downloader/api.go @@ -25,21 +25,21 @@ import ( "github.com/ethereum/go-ethereum/rpc" ) -// PublicDownloaderAPI provides an API which gives information about the current synchronisation status. +// DownloaderAPI provides an API which gives information about the current synchronisation status. // It offers only methods that operates on data that can be available to anyone without security risks. -type PublicDownloaderAPI struct { +type DownloaderAPI struct { d *Downloader mux *event.TypeMux installSyncSubscription chan chan interface{} uninstallSyncSubscription chan *uninstallSyncSubscriptionRequest } -// NewPublicDownloaderAPI create a new PublicDownloaderAPI. The API has an internal event loop that +// NewDownloaderAPI create a new DownloaderAPI. The API has an internal event loop that // listens for events from the downloader through the global event mux. In case it receives one of // these events it broadcasts it to all syncing subscriptions that are installed through the // installSyncSubscription channel. -func NewPublicDownloaderAPI(d *Downloader, m *event.TypeMux) *PublicDownloaderAPI { - api := &PublicDownloaderAPI{ +func NewDownloaderAPI(d *Downloader, m *event.TypeMux) *DownloaderAPI { + api := &DownloaderAPI{ d: d, mux: m, installSyncSubscription: make(chan chan interface{}), @@ -53,7 +53,7 @@ func NewPublicDownloaderAPI(d *Downloader, m *event.TypeMux) *PublicDownloaderAP // eventLoop runs a loop until the event mux closes. It will install and uninstall new // sync subscriptions and broadcasts sync status updates to the installed sync subscriptions. -func (api *PublicDownloaderAPI) eventLoop() { +func (api *DownloaderAPI) eventLoop() { var ( sub = api.mux.Subscribe(StartEvent{}, DoneEvent{}, FailedEvent{}) syncSubscriptions = make(map[chan interface{}]struct{}) @@ -90,7 +90,7 @@ func (api *PublicDownloaderAPI) eventLoop() { } // Syncing provides information when this nodes starts synchronising with the Ethereum network and when it's finished. -func (api *PublicDownloaderAPI) Syncing(ctx context.Context) (*rpc.Subscription, error) { +func (api *DownloaderAPI) Syncing(ctx context.Context) (*rpc.Subscription, error) { notifier, supported := rpc.NotifierFromContext(ctx) if !supported { return &rpc.Subscription{}, rpc.ErrNotificationsUnsupported @@ -125,7 +125,7 @@ type SyncingResult struct { Status ethereum.SyncProgress `json:"status"` } -// uninstallSyncSubscriptionRequest uninstalles a syncing subscription in the API event loop. +// uninstallSyncSubscriptionRequest uninstalls a syncing subscription in the API event loop. type uninstallSyncSubscriptionRequest struct { c chan interface{} uninstalled chan interface{} @@ -133,9 +133,9 @@ type uninstallSyncSubscriptionRequest struct { // SyncStatusSubscription represents a syncing subscription. type SyncStatusSubscription struct { - api *PublicDownloaderAPI // register subscription in event loop of this api instance - c chan interface{} // channel where events are broadcasted to - unsubOnce sync.Once // make sure unsubscribe logic is executed once + api *DownloaderAPI // register subscription in event loop of this api instance + c chan interface{} // channel where events are broadcasted to + unsubOnce sync.Once // make sure unsubscribe logic is executed once } // Unsubscribe uninstalls the subscription from the DownloadAPI event loop. @@ -159,8 +159,8 @@ func (s *SyncStatusSubscription) Unsubscribe() { } // SubscribeSyncStatus creates a subscription that will broadcast new synchronisation updates. -// The given channel must receive interface values, the result can either -func (api *PublicDownloaderAPI) SubscribeSyncStatus(status chan interface{}) *SyncStatusSubscription { +// The given channel must receive interface values, the result can either. +func (api *DownloaderAPI) SubscribeSyncStatus(status chan interface{}) *SyncStatusSubscription { api.installSyncSubscription <- status return &SyncStatusSubscription{api: api, c: status} } diff --git a/eth/downloader/beaconsync.go b/eth/downloader/beaconsync.go index 533404f6c9b9b..484a4e20de64a 100644 --- a/eth/downloader/beaconsync.go +++ b/eth/downloader/beaconsync.go @@ -137,6 +137,13 @@ func (b *beaconBackfiller) setMode(mode SyncMode) { b.resume() } +// SetBadBlockCallback sets the callback to run when a bad block is hit by the +// block processor. This method is not thread safe and should be set only once +// on startup before system events are fired. +func (d *Downloader) SetBadBlockCallback(onBadBlock badBlockFn) { + d.badBlock = onBadBlock +} + // BeaconSync is the post-merge version of the chain synchronization, where the // chain is not downloaded from genesis onward, rather from trusted head announces // backwards. @@ -229,7 +236,7 @@ func (d *Downloader) findBeaconAncestor() (uint64, error) { // Binary search to find the ancestor start, end := beaconTail.Number.Uint64()-1, number if number := beaconHead.Number.Uint64(); end > number { - // This shouldn't really happen in a healty network, but if the consensus + // This shouldn't really happen in a healthy network, but if the consensus // clients feeds us a shorter chain as the canonical, we should not attempt // to access non-existent skeleton items. log.Warn("Beacon head lower than local chain", "beacon", number, "local", end) diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index c836fdd4b8cf1..c04352f0aac63 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -85,6 +85,10 @@ var ( // peerDropFn is a callback type for dropping a peer detected as malicious. type peerDropFn func(id string) +// badBlockFn is a callback for the async beacon sync to notify the caller that +// the origin header requested to sync to, produced a chain with a bad block. +type badBlockFn func(invalid *types.Header, origin *types.Header) + // headerTask is a set of downloaded headers to queue along with their precomputed // hashes to avoid constant rehashing. type headerTask struct { @@ -113,6 +117,7 @@ type Downloader struct { // Callbacks dropPeer peerDropFn // Drops a peer for misbehaving + badBlock badBlockFn // Reports a block as rejected by the chain // Status synchroniseMock func(id string, hash common.Hash) error // Replacement for synchronise during testing @@ -131,7 +136,6 @@ type Downloader struct { pivotHeader *types.Header // Pivot block header to dynamically push the syncing state root pivotLock sync.RWMutex // Lock protecting pivot header reads from updates - snapSync bool // Whether to run state sync over the snap protocol SnapSyncer *snap.Syncer // TODO(karalabe): make private! hack for now stateSyncStart chan *stateSync @@ -360,7 +364,7 @@ func (d *Downloader) synchronise(id string, hash common.Hash, td, ttd *big.Int, // The beacon header syncer is async. It will start this synchronization and // will continue doing other tasks. However, if synchronization needs to be // cancelled, the syncer needs to know if we reached the startup point (and - // inited the cancel cannel) or not yet. Make sure that we'll signal even in + // inited the cancel channel) or not yet. Make sure that we'll signal even in // case of a failure. if beaconPing != nil { defer func() { @@ -1457,7 +1461,7 @@ func (d *Downloader) processHeaders(origin uint64, td, ttd *big.Int, beaconMode } d.syncStatsLock.Unlock() - // Signal the content downloaders of the availablility of new tasks + // Signal the content downloaders of the availability of new tasks for _, ch := range []chan bool{d.queue.blockWakeCh, d.queue.receiptWakeCh} { select { case ch <- true: @@ -1529,7 +1533,7 @@ func (d *Downloader) importBlockResults(results []*fetchResult) error { return errCancelContentProcessing default: } - // Retrieve the a batch of results to import + // Retrieve a batch of results to import first, last := results[0].Header, results[len(results)-1].Header log.Debug("Inserting downloaded chain", "items", len(results), "firstnum", first.Number, "firsthash", first.Hash(), @@ -1545,6 +1549,16 @@ func (d *Downloader) importBlockResults(results []*fetchResult) error { if index, err := d.blockchain.InsertChain(blocks); err != nil { if index < len(results) { log.Debug("Downloaded item processing failed", "number", results[index].Header.Number, "hash", results[index].Header.Hash(), "err", err) + + // In post-merge, notify the engine API of encountered bad chains + if d.badBlock != nil { + head, _, err := d.skeleton.Bounds() + if err != nil { + log.Error("Failed to retrieve beacon bounds for bad block reporting", "err", err) + } else { + d.badBlock(blocks[index].Header(), head) + } + } } else { // The InsertChain method in blockchain.go will sometimes return an out-of-bounds index, // when it needs to preprocess blocks to import a sidechain. diff --git a/eth/downloader/downloader_test.go b/eth/downloader/downloader_test.go index a5db037a456ca..450ed61efc5db 100644 --- a/eth/downloader/downloader_test.go +++ b/eth/downloader/downloader_test.go @@ -68,7 +68,11 @@ func newTesterWithNotification(t *testing.T, success func()) *downloadTester { t.Cleanup(func() { db.Close() }) - core.GenesisBlockForTesting(db, testAddress, big.NewInt(1000000000000000)) + gspec := core.Genesis{ + Alloc: core.GenesisAlloc{testAddress: {Balance: big.NewInt(1000000000000000)}}, + BaseFee: big.NewInt(params.InitialBaseFee), + } + gspec.MustCommit(db) chain, err := core.NewBlockChain(db, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil) if err != nil { @@ -356,7 +360,7 @@ func (dlp *downloadTesterPeer) RequestAccountRange(id uint64, root, origin, limi } // RequestStorageRanges fetches a batch of storage slots belonging to one or -// more accounts. If slots from only one accout is requested, an origin marker +// more accounts. If slots from only one account is requested, an origin marker // may also be used to retrieve from there. func (dlp *downloadTesterPeer) RequestStorageRanges(id uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, bytes uint64) error { // Create the request and service it @@ -395,7 +399,7 @@ func (dlp *downloadTesterPeer) RequestByteCodes(id uint64, hashes []common.Hash, } // RequestTrieNodes fetches a batch of account or storage trie nodes rooted in -// a specificstate trie. +// a specific state trie. func (dlp *downloadTesterPeer) RequestTrieNodes(id uint64, root common.Hash, paths []snap.TrieNodePathSet, bytes uint64) error { req := &snap.GetTrieNodesPacket{ ID: id, @@ -437,6 +441,9 @@ func assertOwnChain(t *testing.T, tester *downloadTester, length int) { func TestCanonicalSynchronisation66Full(t *testing.T) { testCanonSync(t, eth.ETH66, FullSync) } func TestCanonicalSynchronisation66Snap(t *testing.T) { testCanonSync(t, eth.ETH66, SnapSync) } func TestCanonicalSynchronisation66Light(t *testing.T) { testCanonSync(t, eth.ETH66, LightSync) } +func TestCanonicalSynchronisation67Full(t *testing.T) { testCanonSync(t, eth.ETH67, FullSync) } +func TestCanonicalSynchronisation67Snap(t *testing.T) { testCanonSync(t, eth.ETH67, SnapSync) } +func TestCanonicalSynchronisation67Light(t *testing.T) { testCanonSync(t, eth.ETH67, LightSync) } func testCanonSync(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -457,6 +464,8 @@ func testCanonSync(t *testing.T, protocol uint, mode SyncMode) { // until the cached blocks are retrieved. func TestThrottling66Full(t *testing.T) { testThrottling(t, eth.ETH66, FullSync) } func TestThrottling66Snap(t *testing.T) { testThrottling(t, eth.ETH66, SnapSync) } +func TestThrottling67Full(t *testing.T) { testThrottling(t, eth.ETH67, FullSync) } +func TestThrottling67Snap(t *testing.T) { testThrottling(t, eth.ETH67, SnapSync) } func testThrottling(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -537,6 +546,9 @@ func testThrottling(t *testing.T, protocol uint, mode SyncMode) { func TestForkedSync66Full(t *testing.T) { testForkedSync(t, eth.ETH66, FullSync) } func TestForkedSync66Snap(t *testing.T) { testForkedSync(t, eth.ETH66, SnapSync) } func TestForkedSync66Light(t *testing.T) { testForkedSync(t, eth.ETH66, LightSync) } +func TestForkedSync67Full(t *testing.T) { testForkedSync(t, eth.ETH67, FullSync) } +func TestForkedSync67Snap(t *testing.T) { testForkedSync(t, eth.ETH67, SnapSync) } +func TestForkedSync67Light(t *testing.T) { testForkedSync(t, eth.ETH67, LightSync) } func testForkedSync(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -559,11 +571,14 @@ func testForkedSync(t *testing.T, protocol uint, mode SyncMode) { assertOwnChain(t, tester, len(chainB.blocks)) } -// Tests that synchronising against a much shorter but much heavyer fork works -// corrently and is not dropped. +// Tests that synchronising against a much shorter but much heavier fork works +// currently and is not dropped. func TestHeavyForkedSync66Full(t *testing.T) { testHeavyForkedSync(t, eth.ETH66, FullSync) } func TestHeavyForkedSync66Snap(t *testing.T) { testHeavyForkedSync(t, eth.ETH66, SnapSync) } func TestHeavyForkedSync66Light(t *testing.T) { testHeavyForkedSync(t, eth.ETH66, LightSync) } +func TestHeavyForkedSync67Full(t *testing.T) { testHeavyForkedSync(t, eth.ETH67, FullSync) } +func TestHeavyForkedSync67Snap(t *testing.T) { testHeavyForkedSync(t, eth.ETH67, SnapSync) } +func TestHeavyForkedSync67Light(t *testing.T) { testHeavyForkedSync(t, eth.ETH67, LightSync) } func testHeavyForkedSync(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -593,6 +608,9 @@ func testHeavyForkedSync(t *testing.T, protocol uint, mode SyncMode) { func TestBoundedForkedSync66Full(t *testing.T) { testBoundedForkedSync(t, eth.ETH66, FullSync) } func TestBoundedForkedSync66Snap(t *testing.T) { testBoundedForkedSync(t, eth.ETH66, SnapSync) } func TestBoundedForkedSync66Light(t *testing.T) { testBoundedForkedSync(t, eth.ETH66, LightSync) } +func TestBoundedForkedSync67Full(t *testing.T) { testBoundedForkedSync(t, eth.ETH67, FullSync) } +func TestBoundedForkedSync67Snap(t *testing.T) { testBoundedForkedSync(t, eth.ETH67, SnapSync) } +func TestBoundedForkedSync67Light(t *testing.T) { testBoundedForkedSync(t, eth.ETH67, LightSync) } func testBoundedForkedSync(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -627,6 +645,15 @@ func TestBoundedHeavyForkedSync66Snap(t *testing.T) { func TestBoundedHeavyForkedSync66Light(t *testing.T) { testBoundedHeavyForkedSync(t, eth.ETH66, LightSync) } +func TestBoundedHeavyForkedSync67Full(t *testing.T) { + testBoundedHeavyForkedSync(t, eth.ETH67, FullSync) +} +func TestBoundedHeavyForkedSync67Snap(t *testing.T) { + testBoundedHeavyForkedSync(t, eth.ETH67, SnapSync) +} +func TestBoundedHeavyForkedSync67Light(t *testing.T) { + testBoundedHeavyForkedSync(t, eth.ETH67, LightSync) +} func testBoundedHeavyForkedSync(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -654,6 +681,9 @@ func testBoundedHeavyForkedSync(t *testing.T, protocol uint, mode SyncMode) { func TestCancel66Full(t *testing.T) { testCancel(t, eth.ETH66, FullSync) } func TestCancel66Snap(t *testing.T) { testCancel(t, eth.ETH66, SnapSync) } func TestCancel66Light(t *testing.T) { testCancel(t, eth.ETH66, LightSync) } +func TestCancel67Full(t *testing.T) { testCancel(t, eth.ETH67, FullSync) } +func TestCancel67Snap(t *testing.T) { testCancel(t, eth.ETH67, SnapSync) } +func TestCancel67Light(t *testing.T) { testCancel(t, eth.ETH67, LightSync) } func testCancel(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -681,6 +711,9 @@ func testCancel(t *testing.T, protocol uint, mode SyncMode) { func TestMultiSynchronisation66Full(t *testing.T) { testMultiSynchronisation(t, eth.ETH66, FullSync) } func TestMultiSynchronisation66Snap(t *testing.T) { testMultiSynchronisation(t, eth.ETH66, SnapSync) } func TestMultiSynchronisation66Light(t *testing.T) { testMultiSynchronisation(t, eth.ETH66, LightSync) } +func TestMultiSynchronisation67Full(t *testing.T) { testMultiSynchronisation(t, eth.ETH67, FullSync) } +func TestMultiSynchronisation67Snap(t *testing.T) { testMultiSynchronisation(t, eth.ETH67, SnapSync) } +func TestMultiSynchronisation67Light(t *testing.T) { testMultiSynchronisation(t, eth.ETH67, LightSync) } func testMultiSynchronisation(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -705,6 +738,9 @@ func testMultiSynchronisation(t *testing.T, protocol uint, mode SyncMode) { func TestMultiProtoSynchronisation66Full(t *testing.T) { testMultiProtoSync(t, eth.ETH66, FullSync) } func TestMultiProtoSynchronisation66Snap(t *testing.T) { testMultiProtoSync(t, eth.ETH66, SnapSync) } func TestMultiProtoSynchronisation66Light(t *testing.T) { testMultiProtoSync(t, eth.ETH66, LightSync) } +func TestMultiProtoSynchronisation67Full(t *testing.T) { testMultiProtoSync(t, eth.ETH67, FullSync) } +func TestMultiProtoSynchronisation67Snap(t *testing.T) { testMultiProtoSync(t, eth.ETH67, SnapSync) } +func TestMultiProtoSynchronisation67Light(t *testing.T) { testMultiProtoSync(t, eth.ETH67, LightSync) } func testMultiProtoSync(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -715,7 +751,7 @@ func testMultiProtoSync(t *testing.T, protocol uint, mode SyncMode) { // Create peers of every type tester.newPeer("peer 66", eth.ETH66, chain.blocks[1:]) - //tester.newPeer("peer 65", eth.ETH67, chain.blocks[1:) + tester.newPeer("peer 67", eth.ETH67, chain.blocks[1:]) // Synchronise with the requested peer and make sure all blocks were retrieved if err := tester.sync(fmt.Sprintf("peer %d", protocol), nil, mode); err != nil { @@ -724,7 +760,7 @@ func testMultiProtoSync(t *testing.T, protocol uint, mode SyncMode) { assertOwnChain(t, tester, len(chain.blocks)) // Check that no peers have been dropped off - for _, version := range []int{66} { + for _, version := range []int{66, 67} { peer := fmt.Sprintf("peer %d", version) if _, ok := tester.peers[peer]; !ok { t.Errorf("%s dropped", peer) @@ -737,6 +773,9 @@ func testMultiProtoSync(t *testing.T, protocol uint, mode SyncMode) { func TestEmptyShortCircuit66Full(t *testing.T) { testEmptyShortCircuit(t, eth.ETH66, FullSync) } func TestEmptyShortCircuit66Snap(t *testing.T) { testEmptyShortCircuit(t, eth.ETH66, SnapSync) } func TestEmptyShortCircuit66Light(t *testing.T) { testEmptyShortCircuit(t, eth.ETH66, LightSync) } +func TestEmptyShortCircuit67Full(t *testing.T) { testEmptyShortCircuit(t, eth.ETH67, FullSync) } +func TestEmptyShortCircuit67Snap(t *testing.T) { testEmptyShortCircuit(t, eth.ETH67, SnapSync) } +func TestEmptyShortCircuit67Light(t *testing.T) { testEmptyShortCircuit(t, eth.ETH67, LightSync) } func testEmptyShortCircuit(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -785,6 +824,9 @@ func testEmptyShortCircuit(t *testing.T, protocol uint, mode SyncMode) { func TestMissingHeaderAttack66Full(t *testing.T) { testMissingHeaderAttack(t, eth.ETH66, FullSync) } func TestMissingHeaderAttack66Snap(t *testing.T) { testMissingHeaderAttack(t, eth.ETH66, SnapSync) } func TestMissingHeaderAttack66Light(t *testing.T) { testMissingHeaderAttack(t, eth.ETH66, LightSync) } +func TestMissingHeaderAttack67Full(t *testing.T) { testMissingHeaderAttack(t, eth.ETH67, FullSync) } +func TestMissingHeaderAttack67Snap(t *testing.T) { testMissingHeaderAttack(t, eth.ETH67, SnapSync) } +func TestMissingHeaderAttack67Light(t *testing.T) { testMissingHeaderAttack(t, eth.ETH67, LightSync) } func testMissingHeaderAttack(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -811,6 +853,9 @@ func testMissingHeaderAttack(t *testing.T, protocol uint, mode SyncMode) { func TestShiftedHeaderAttack66Full(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH66, FullSync) } func TestShiftedHeaderAttack66Snap(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH66, SnapSync) } func TestShiftedHeaderAttack66Light(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH66, LightSync) } +func TestShiftedHeaderAttack67Full(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH67, FullSync) } +func TestShiftedHeaderAttack67Snap(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH67, SnapSync) } +func TestShiftedHeaderAttack67Light(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH67, LightSync) } func testShiftedHeaderAttack(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -837,6 +882,7 @@ func testShiftedHeaderAttack(t *testing.T, protocol uint, mode SyncMode) { // for various failure scenarios. Afterwards a full sync is attempted to make // sure no state was corrupted. func TestInvalidHeaderRollback66Snap(t *testing.T) { testInvalidHeaderRollback(t, eth.ETH66, SnapSync) } +func TestInvalidHeaderRollback67Snap(t *testing.T) { testInvalidHeaderRollback(t, eth.ETH67, SnapSync) } func testInvalidHeaderRollback(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -923,6 +969,15 @@ func TestHighTDStarvationAttack66Snap(t *testing.T) { func TestHighTDStarvationAttack66Light(t *testing.T) { testHighTDStarvationAttack(t, eth.ETH66, LightSync) } +func TestHighTDStarvationAttack67Full(t *testing.T) { + testHighTDStarvationAttack(t, eth.ETH67, FullSync) +} +func TestHighTDStarvationAttack67Snap(t *testing.T) { + testHighTDStarvationAttack(t, eth.ETH67, SnapSync) +} +func TestHighTDStarvationAttack67Light(t *testing.T) { + testHighTDStarvationAttack(t, eth.ETH67, LightSync) +} func testHighTDStarvationAttack(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -937,6 +992,7 @@ func testHighTDStarvationAttack(t *testing.T, protocol uint, mode SyncMode) { // Tests that misbehaving peers are disconnected, whilst behaving ones are not. func TestBlockHeaderAttackerDropping66(t *testing.T) { testBlockHeaderAttackerDropping(t, eth.ETH66) } +func TestBlockHeaderAttackerDropping67(t *testing.T) { testBlockHeaderAttackerDropping(t, eth.ETH67) } func testBlockHeaderAttackerDropping(t *testing.T, protocol uint) { // Define the disconnection requirement for individual hash fetch errors @@ -987,6 +1043,9 @@ func testBlockHeaderAttackerDropping(t *testing.T, protocol uint) { func TestSyncProgress66Full(t *testing.T) { testSyncProgress(t, eth.ETH66, FullSync) } func TestSyncProgress66Snap(t *testing.T) { testSyncProgress(t, eth.ETH66, SnapSync) } func TestSyncProgress66Light(t *testing.T) { testSyncProgress(t, eth.ETH66, LightSync) } +func TestSyncProgress67Full(t *testing.T) { testSyncProgress(t, eth.ETH67, FullSync) } +func TestSyncProgress67Snap(t *testing.T) { testSyncProgress(t, eth.ETH67, SnapSync) } +func TestSyncProgress67Light(t *testing.T) { testSyncProgress(t, eth.ETH67, LightSync) } func testSyncProgress(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -1064,6 +1123,9 @@ func checkProgress(t *testing.T, d *Downloader, stage string, want ethereum.Sync func TestForkedSyncProgress66Full(t *testing.T) { testForkedSyncProgress(t, eth.ETH66, FullSync) } func TestForkedSyncProgress66Snap(t *testing.T) { testForkedSyncProgress(t, eth.ETH66, SnapSync) } func TestForkedSyncProgress66Light(t *testing.T) { testForkedSyncProgress(t, eth.ETH66, LightSync) } +func TestForkedSyncProgress67Full(t *testing.T) { testForkedSyncProgress(t, eth.ETH67, FullSync) } +func TestForkedSyncProgress67Snap(t *testing.T) { testForkedSyncProgress(t, eth.ETH67, SnapSync) } +func TestForkedSyncProgress67Light(t *testing.T) { testForkedSyncProgress(t, eth.ETH67, LightSync) } func testForkedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -1135,6 +1197,9 @@ func testForkedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { func TestFailedSyncProgress66Full(t *testing.T) { testFailedSyncProgress(t, eth.ETH66, FullSync) } func TestFailedSyncProgress66Snap(t *testing.T) { testFailedSyncProgress(t, eth.ETH66, SnapSync) } func TestFailedSyncProgress66Light(t *testing.T) { testFailedSyncProgress(t, eth.ETH66, LightSync) } +func TestFailedSyncProgress67Full(t *testing.T) { testFailedSyncProgress(t, eth.ETH67, FullSync) } +func TestFailedSyncProgress67Snap(t *testing.T) { testFailedSyncProgress(t, eth.ETH67, SnapSync) } +func TestFailedSyncProgress67Light(t *testing.T) { testFailedSyncProgress(t, eth.ETH67, LightSync) } func testFailedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -1201,6 +1266,9 @@ func testFailedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { func TestFakedSyncProgress66Full(t *testing.T) { testFakedSyncProgress(t, eth.ETH66, FullSync) } func TestFakedSyncProgress66Snap(t *testing.T) { testFakedSyncProgress(t, eth.ETH66, SnapSync) } func TestFakedSyncProgress66Light(t *testing.T) { testFakedSyncProgress(t, eth.ETH66, LightSync) } +func TestFakedSyncProgress67Full(t *testing.T) { testFakedSyncProgress(t, eth.ETH67, FullSync) } +func TestFakedSyncProgress67Snap(t *testing.T) { testFakedSyncProgress(t, eth.ETH67, SnapSync) } +func TestFakedSyncProgress67Light(t *testing.T) { testFakedSyncProgress(t, eth.ETH67, LightSync) } func testFakedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -1347,6 +1415,11 @@ func TestCheckpointEnforcement66Snap(t *testing.T) { testCheckpointEnforcement(t func TestCheckpointEnforcement66Light(t *testing.T) { testCheckpointEnforcement(t, eth.ETH66, LightSync) } +func TestCheckpointEnforcement67Full(t *testing.T) { testCheckpointEnforcement(t, eth.ETH67, FullSync) } +func TestCheckpointEnforcement67Snap(t *testing.T) { testCheckpointEnforcement(t, eth.ETH67, SnapSync) } +func TestCheckpointEnforcement67Light(t *testing.T) { + testCheckpointEnforcement(t, eth.ETH67, LightSync) +} func testCheckpointEnforcement(t *testing.T, protocol uint, mode SyncMode) { // Create a new tester with a particular hard coded checkpoint block diff --git a/eth/downloader/fetchers_concurrent.go b/eth/downloader/fetchers_concurrent.go index a0aa197175a31..44e6aa8f8d88d 100644 --- a/eth/downloader/fetchers_concurrent.go +++ b/eth/downloader/fetchers_concurrent.go @@ -47,7 +47,7 @@ type typedQueue interface { // capacity is responsible for calculating how many items of the abstracted // type a particular peer is estimated to be able to retrieve within the - // alloted round trip time. + // allotted round trip time. capacity(peer *peerConnection, rtt time.Duration) int // updateCapacity is responsible for updating how many items of the abstracted @@ -58,7 +58,7 @@ type typedQueue interface { // from the download queue to the specified peer. reserve(peer *peerConnection, items int) (*fetchRequest, bool, bool) - // unreserve is resposible for removing the current retrieval allocation + // unreserve is responsible for removing the current retrieval allocation // assigned to a specific peer and placing it back into the pool to allow // reassigning to some other peer. unreserve(peer string) int @@ -190,7 +190,7 @@ func (d *Downloader) concurrentFetch(queue typedQueue, beaconMode bool) error { req, err := queue.request(peer, request, responses) if err != nil { // Sending the request failed, which generally means the peer - // was diconnected in between assignment and network send. + // was disconnected in between assignment and network send. // Although all peer removal operations return allocated tasks // to the queue, that is async, and we can do better here by // immediately pushing the unfulfilled requests. diff --git a/eth/downloader/fetchers_concurrent_bodies.go b/eth/downloader/fetchers_concurrent_bodies.go index a8de410323f33..e84206fe99519 100644 --- a/eth/downloader/fetchers_concurrent_bodies.go +++ b/eth/downloader/fetchers_concurrent_bodies.go @@ -41,7 +41,7 @@ func (q *bodyQueue) pending() int { } // capacity is responsible for calculating how many bodies a particular peer is -// estimated to be able to retrieve within the alloted round trip time. +// estimated to be able to retrieve within the allotted round trip time. func (q *bodyQueue) capacity(peer *peerConnection, rtt time.Duration) int { return peer.BodyCapacity(rtt) } @@ -58,7 +58,7 @@ func (q *bodyQueue) reserve(peer *peerConnection, items int) (*fetchRequest, boo return q.queue.ReserveBodies(peer, items) } -// unreserve is resposible for removing the current body retrieval allocation +// unreserve is responsible for removing the current body retrieval allocation // assigned to a specific peer and placing it back into the pool to allow // reassigning to some other peer. func (q *bodyQueue) unreserve(peer string) int { diff --git a/eth/downloader/fetchers_concurrent_headers.go b/eth/downloader/fetchers_concurrent_headers.go index bd3bb3e00bf37..84c7f209865a9 100644 --- a/eth/downloader/fetchers_concurrent_headers.go +++ b/eth/downloader/fetchers_concurrent_headers.go @@ -41,7 +41,7 @@ func (q *headerQueue) pending() int { } // capacity is responsible for calculating how many headers a particular peer is -// estimated to be able to retrieve within the alloted round trip time. +// estimated to be able to retrieve within the allotted round trip time. func (q *headerQueue) capacity(peer *peerConnection, rtt time.Duration) int { return peer.HeaderCapacity(rtt) } @@ -58,7 +58,7 @@ func (q *headerQueue) reserve(peer *peerConnection, items int) (*fetchRequest, b return q.queue.ReserveHeaders(peer, items), false, false } -// unreserve is resposible for removing the current header retrieval allocation +// unreserve is responsible for removing the current header retrieval allocation // assigned to a specific peer and placing it back into the pool to allow // reassigning to some other peer. func (q *headerQueue) unreserve(peer string) int { diff --git a/eth/downloader/fetchers_concurrent_receipts.go b/eth/downloader/fetchers_concurrent_receipts.go index fee2c34101d27..1c853c2184432 100644 --- a/eth/downloader/fetchers_concurrent_receipts.go +++ b/eth/downloader/fetchers_concurrent_receipts.go @@ -28,7 +28,7 @@ import ( // concurrent fetcher and the downloader. type receiptQueue Downloader -// waker returns a notification channel that gets pinged in case more reecipt +// waker returns a notification channel that gets pinged in case more receipt // fetches have been queued up, so the fetcher might assign it to idle peers. func (q *receiptQueue) waker() chan bool { return q.queue.receiptWakeCh @@ -41,7 +41,7 @@ func (q *receiptQueue) pending() int { } // capacity is responsible for calculating how many receipts a particular peer is -// estimated to be able to retrieve within the alloted round trip time. +// estimated to be able to retrieve within the allotted round trip time. func (q *receiptQueue) capacity(peer *peerConnection, rtt time.Duration) int { return peer.ReceiptCapacity(rtt) } @@ -58,7 +58,7 @@ func (q *receiptQueue) reserve(peer *peerConnection, items int) (*fetchRequest, return q.queue.ReserveReceipts(peer, items) } -// unreserve is resposible for removing the current receipt retrieval allocation +// unreserve is responsible for removing the current receipt retrieval allocation // assigned to a specific peer and placing it back into the pool to allow // reassigning to some other peer. func (q *receiptQueue) unreserve(peer string) int { diff --git a/eth/downloader/peer.go b/eth/downloader/peer.go index d74d23e74d557..6b8269495948e 100644 --- a/eth/downloader/peer.go +++ b/eth/downloader/peer.go @@ -237,6 +237,7 @@ func (ps *peerSet) Register(p *peerConnection) error { } p.rates = msgrate.NewTracker(ps.rates.MeanCapacities(), ps.rates.MedianRoundTrip()) if err := ps.rates.Track(p.id, p.rates); err != nil { + ps.lock.Unlock() return err } ps.peers[p.id] = p diff --git a/eth/downloader/queue.go b/eth/downloader/queue.go index ff34d932f0183..a8d2ea83a9eeb 100644 --- a/eth/downloader/queue.go +++ b/eth/downloader/queue.go @@ -817,7 +817,6 @@ func (q *queue) deliver(id string, taskPool map[common.Hash]*types.Header, reqTimer metrics.Timer, resInMeter metrics.Meter, resDropMeter metrics.Meter, results int, validate func(index int, header *types.Header) error, reconstruct func(index int, result *fetchResult)) (int, error) { - // Short circuit if the data was never requested request := pendPool[id] if request == nil { @@ -860,7 +859,7 @@ func (q *queue) deliver(id string, taskPool map[common.Hash]*types.Header, if res, stale, err := q.resultCache.GetDeliverySlot(header.Number.Uint64()); err == nil { reconstruct(accepted, res) } else { - // else: betweeen here and above, some other peer filled this result, + // else: between here and above, some other peer filled this result, // or it was indeed a no-op. This should not happen, but if it does it's // not something to panic about log.Error("Delivery stale", "stale", stale, "number", header.Number.Uint64(), "err", err) diff --git a/eth/downloader/queue_test.go b/eth/downloader/queue_test.go index 09b18afe5df5f..8631b27c9275a 100644 --- a/eth/downloader/queue_test.go +++ b/eth/downloader/queue_test.go @@ -27,24 +27,18 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/trie" ) -var ( - testdb = rawdb.NewMemoryDatabase() - genesis = core.GenesisBlockForTesting(testdb, testAddress, big.NewInt(1000000000000000)) -) - // makeChain creates a chain of n blocks starting at and including parent. // the returned hash chain is ordered head->parent. In addition, every 3rd block // contains a transaction and every 5th an uncle to allow testing correct block // reassembly. func makeChain(n int, seed byte, parent *types.Block, empty bool) ([]*types.Block, []types.Receipts) { - blocks, receipts := core.GenerateChain(params.TestChainConfig, parent, ethash.NewFaker(), testdb, n, func(i int, block *core.BlockGen) { + blocks, receipts := core.GenerateChain(params.TestChainConfig, parent, ethash.NewFaker(), testDB, n, func(i int, block *core.BlockGen) { block.SetCoinbase(common.Address{seed}) // Add one tx to every secondblock if !empty && i%2 == 0 { @@ -70,10 +64,10 @@ var emptyChain *chainData func init() { // Create a chain of blocks to import targetBlocks := 128 - blocks, _ := makeChain(targetBlocks, 0, genesis, false) + blocks, _ := makeChain(targetBlocks, 0, testGenesis, false) chain = &chainData{blocks, 0} - blocks, _ = makeChain(targetBlocks, 0, genesis, true) + blocks, _ = makeChain(targetBlocks, 0, testGenesis, true) emptyChain = &chainData{blocks, 0} } @@ -156,7 +150,7 @@ func TestBasics(t *testing.T) { // The second peer should hit throttling if !throttle { - t.Fatalf("should not throttle") + t.Fatalf("should throttle") } // And not get any fetches at all, since it was throttled to begin with if fetchReq != nil { @@ -185,7 +179,6 @@ func TestBasics(t *testing.T) { if got, exp := fetchReq.Headers[0].Number.Uint64(), uint64(1); got != exp { t.Fatalf("expected header %d, got %d", exp, got) } - } if exp, got := q.blockTaskQueue.Size(), numOfBlocks-10; exp != got { t.Errorf("expected block task queue to be %d, got %d", exp, got) @@ -239,7 +232,6 @@ func TestEmptyBlocks(t *testing.T) { if fetchReq != nil { t.Fatal("there should be no body fetch tasks remaining") } - } if q.blockTaskQueue.Size() != numOfBlocks-10 { t.Errorf("expected block task queue to be %d, got %d", numOfBlocks-10, q.blockTaskQueue.Size()) @@ -253,7 +245,7 @@ func TestEmptyBlocks(t *testing.T) { // there should be nothing to fetch, blocks are empty if fetchReq != nil { - t.Fatal("there should be no body fetch tasks remaining") + t.Fatal("there should be no receipt fetch tasks remaining") } } if q.blockTaskQueue.Size() != numOfBlocks-10 { @@ -273,14 +265,13 @@ func TestEmptyBlocks(t *testing.T) { // some more advanced scenarios func XTestDelivery(t *testing.T) { // the outside network, holding blocks - blo, rec := makeChain(128, 0, genesis, false) + blo, rec := makeChain(128, 0, testGenesis, false) world := newNetwork() world.receipts = rec world.chain = blo world.progress(10) if false { log.Root().SetHandler(log.StdoutHandler) - } q := newQueue(10, 10) var wg sync.WaitGroup @@ -315,7 +306,6 @@ func XTestDelivery(t *testing.T) { fmt.Printf("got %d results, %d tot\n", len(res), tot) // Now we can forget about these world.forget(res[len(res)-1].Header.Number.Uint64()) - } }() wg.Add(1) @@ -396,7 +386,6 @@ func XTestDelivery(t *testing.T) { } for i := 0; i < 50; i++ { time.Sleep(2990 * time.Millisecond) - } }() wg.Add(1) @@ -447,10 +436,8 @@ func (n *network) forget(blocknum uint64) { n.chain = n.chain[index:] n.receipts = n.receipts[index:] n.offset = int(blocknum) - } func (n *network) progress(numBlocks int) { - n.lock.Lock() defer n.lock.Unlock() //fmt.Printf("progressing...\n") @@ -458,7 +445,6 @@ func (n *network) progress(numBlocks int) { n.chain = append(n.chain, newBlocks...) n.receipts = append(n.receipts, newR...) n.cond.Broadcast() - } func (n *network) headers(from int) []*types.Header { diff --git a/eth/downloader/skeleton.go b/eth/downloader/skeleton.go index be4e8fbfc10cd..517b8378c5181 100644 --- a/eth/downloader/skeleton.go +++ b/eth/downloader/skeleton.go @@ -51,7 +51,7 @@ const requestHeaders = 512 // errSyncLinked is an internal helper error to signal that the current sync // cycle linked up to the genesis block, this the skeleton syncer should ping // the backfiller to resume. Since we already have that logic on sync start, -// piggie-back on that instead of 2 entrypoints. +// piggy-back on that instead of 2 entrypoints. var errSyncLinked = errors.New("sync linked") // errSyncMerged is an internal helper error to signal that the current sync @@ -148,7 +148,7 @@ type backfiller interface { // suspend requests the backfiller to abort any running full or snap sync // based on the skeleton chain as it might be invalid. The backfiller should // gracefully handle multiple consecutive suspends without a resume, even - // on initial sartup. + // on initial startup. // // The method should return the last block header that has been successfully // backfilled, or nil if the backfiller was not resumed. @@ -209,7 +209,7 @@ type skeleton struct { headEvents chan *headUpdate // Notification channel for new heads terminate chan chan error // Termination channel to abort sync - terminated chan struct{} // Channel to signal that the syner is dead + terminated chan struct{} // Channel to signal that the syncer is dead // Callback hooks used during testing syncStarting func() // callback triggered after a sync cycle is inited but before started @@ -358,6 +358,7 @@ func (s *skeleton) sync(head *types.Header) (*types.Header, error) { // If the sync is already done, resume the backfiller. When the loop stops, // terminate the backfiller too. linked := len(s.progress.Subchains) == 1 && + rawdb.HasHeader(s.db, s.progress.Subchains[0].Next, s.scratchHead) && rawdb.HasBody(s.db, s.progress.Subchains[0].Next, s.scratchHead) && rawdb.HasReceipts(s.db, s.progress.Subchains[0].Next, s.scratchHead) if linked { @@ -553,7 +554,7 @@ func (s *skeleton) initSync(head *types.Header) { return } } - // Either we've failed to decode the previus state, or there was none. Start + // Either we've failed to decode the previous state, or there was none. Start // a fresh sync with a single subchain represented by the currently sent // chain head. s.progress = &skeletonProgress{ @@ -823,7 +824,7 @@ func (s *skeleton) executeTask(peer *peerConnection, req *headerRequest) { } } -// revertRequests locates all the currently pending reuqests from a particular +// revertRequests locates all the currently pending requests from a particular // peer and reverts them, rescheduling for others to fulfill. func (s *skeleton) revertRequests(peer string) { // Gather the requests first, revertals need the lock too @@ -871,7 +872,7 @@ func (s *skeleton) revertRequest(req *headerRequest) { delete(s.requests, req.id) // Remove the request from the tracked set and mark the task as not-pending, - // ready for resheduling + // ready for rescheduling s.scratchOwners[(s.scratchHead-req.head)/requestHeaders] = "" } @@ -946,12 +947,12 @@ func (s *skeleton) processResponse(res *headerResponse) (linked bool, merged boo // In the case of full sync it would be enough to check for the body, // but even a full syncing node will generate a receipt once block // processing is done, so it's just one more "needless" check. - var ( - hasBody = rawdb.HasBody(s.db, header.ParentHash, header.Number.Uint64()-1) - hasReceipt = rawdb.HasReceipts(s.db, header.ParentHash, header.Number.Uint64()-1) - ) - if hasBody && hasReceipt { - linked = true + // + // The weird cascading checks are done to minimize the database reads. + linked = rawdb.HasHeader(s.db, header.ParentHash, header.Number.Uint64()-1) && + rawdb.HasBody(s.db, header.ParentHash, header.Number.Uint64()-1) && + rawdb.HasReceipts(s.db, header.ParentHash, header.Number.Uint64()-1) + if linked { break } } diff --git a/eth/downloader/skeleton_test.go b/eth/downloader/skeleton_test.go index 836efabebcb85..6bcbac3a89feb 100644 --- a/eth/downloader/skeleton_test.go +++ b/eth/downloader/skeleton_test.go @@ -21,7 +21,6 @@ import ( "errors" "fmt" "math/big" - "os" "sync/atomic" "testing" "time" @@ -54,7 +53,7 @@ func newHookedBackfiller() backfiller { // suspend requests the backfiller to abort any running full or snap sync // based on the skeleton chain as it might be invalid. The backfiller should // gracefully handle multiple consecutive suspends without a resume, even -// on initial sartup. +// on initial startup. func (hf *hookedBackfiller) suspend() *types.Header { if hf.suspendHook != nil { hf.suspendHook() @@ -112,7 +111,7 @@ func newSkeletonTestPeerWithHook(id string, headers []*types.Header, serve func( // function can be used to retrieve batches of headers from the particular peer. func (p *skeletonTestPeer) RequestHeadersByNumber(origin uint64, amount int, skip int, reverse bool, sink chan *eth.Response) (*eth.Request, error) { // Since skeleton test peer are in-memory mocks, dropping the does not make - // them inaccepssible. As such, check a local `dropped` field to see if the + // them inaccessible. As such, check a local `dropped` field to see if the // peer has been dropped and should not respond any more. if atomic.LoadUint64(&p.dropped) != 0 { return nil, errors.New("peer already dropped") @@ -205,7 +204,7 @@ func (p *skeletonTestPeer) RequestReceipts([]common.Hash, chan *eth.Response) (* panic("skeleton sync must not request receipts") } -// Tests various sync initialzations based on previous leftovers in the database +// Tests various sync initializations based on previous leftovers in the database // and announced heads. func TestSkeletonSyncInit(t *testing.T) { // Create a few key headers @@ -228,7 +227,7 @@ func TestSkeletonSyncInit(t *testing.T) { newstate: []*subchain{{Head: 50, Tail: 50}}, }, // Empty database with only the genesis set with a leftover empty sync - // progess. This is a synthetic case, just for the sake of covering things. + // progress. This is a synthetic case, just for the sake of covering things. { oldstate: []*subchain{}, head: block50, @@ -515,7 +514,7 @@ func TestSkeletonSyncExtend(t *testing.T) { // Tests that the skeleton sync correctly retrieves headers from one or more // peers without duplicates or other strange side effects. func TestSkeletonSyncRetrievals(t *testing.T) { - log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true)))) + //log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true)))) // Since skeleton headers don't need to be meaningful, beyond a parent hash // progression, create a long fake chain to test with. @@ -534,13 +533,13 @@ func TestSkeletonSyncRetrievals(t *testing.T) { peers []*skeletonTestPeer // Initial peer set to start the sync with midstate []*subchain // Expected sync state after initial cycle midserve uint64 // Expected number of header retrievals after initial cycle - middrop uint64 // Expectd number of peers dropped after initial cycle + middrop uint64 // Expected number of peers dropped after initial cycle - newHead *types.Header // New header to annount on top of the old one + newHead *types.Header // New header to anoint on top of the old one newPeer *skeletonTestPeer // New peer to join the skeleton syncer endstate []*subchain // Expected sync state after the post-init event endserve uint64 // Expected number of header retrievals after the post-init event - enddrop uint64 // Expectd number of peers dropped after the post-init event + enddrop uint64 // Expected number of peers dropped after the post-init event }{ // Completely empty database with only the genesis set. The sync is expected // to create a single subchain with the requested head. No peers however, so @@ -791,7 +790,6 @@ func TestSkeletonSyncRetrievals(t *testing.T) { check := func() error { if len(progress.Subchains) != len(tt.midstate) { return fmt.Errorf("test %d, mid state: subchain count mismatch: have %d, want %d", i, len(progress.Subchains), len(tt.midstate)) - } for j := 0; j < len(progress.Subchains); j++ { if progress.Subchains[j].Head != tt.midstate[j].Head { diff --git a/eth/downloader/testchain_test.go b/eth/downloader/testchain_test.go index 8b873343cac46..785da40b59895 100644 --- a/eth/downloader/testchain_test.go +++ b/eth/downloader/testchain_test.go @@ -37,7 +37,12 @@ var ( testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") testAddress = crypto.PubkeyToAddress(testKey.PublicKey) testDB = rawdb.NewMemoryDatabase() - testGenesis = core.GenesisBlockForTesting(testDB, testAddress, big.NewInt(1000000000000000)) + + testGspec = core.Genesis{ + Alloc: core.GenesisAlloc{testAddress: {Balance: big.NewInt(1000000000000000)}}, + BaseFee: big.NewInt(params.InitialBaseFee), + } + testGenesis = testGspec.MustCommit(testDB) ) // The common prefix of all test chains: @@ -212,7 +217,7 @@ func newTestBlockchain(blocks []*types.Block) *core.BlockChain { panic("Requested chain generation outside of init") } db := rawdb.NewMemoryDatabase() - core.GenesisBlockForTesting(db, testAddress, big.NewInt(1000000000000000)) + testGspec.MustCommit(db) chain, err := core.NewBlockChain(db, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil) if err != nil { diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index ca29aad8f0803..5690366421dcb 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -83,6 +83,7 @@ var Defaults = Config{ TrieDirtyCache: 256, TrieTimeout: 60 * time.Minute, SnapshotCache: 102, + FilterLogCacheSize: 32, Miner: miner.Config{ GasCeil: 30000000, GasPrice: big.NewInt(params.GWei), @@ -171,6 +172,9 @@ type Config struct { SnapshotCache int Preimages bool + // This is the number of blocks for which logs will be cached in the filter system. + FilterLogCacheSize int + // Mining options Miner miner.Config @@ -196,7 +200,7 @@ type Config struct { RPCEVMTimeout time.Duration // RPCTxFeeCap is the global transaction fee(price * gaslimit) cap for - // send-transction variants. The unit is ether. + // send-transaction variants. The unit is ether. RPCTxFeeCap float64 // Checkpoint is a hardcoded checkpoint which can be nil. @@ -205,11 +209,11 @@ type Config struct { // CheckpointOracle is the configuration for checkpoint oracle. CheckpointOracle *params.CheckpointOracleConfig `toml:",omitempty"` - // Arrow Glacier block override (TODO: remove after the fork) - OverrideArrowGlacier *big.Int `toml:",omitempty"` - // OverrideTerminalTotalDifficulty (TODO: remove after the fork) OverrideTerminalTotalDifficulty *big.Int `toml:",omitempty"` + + // OverrideTerminalTotalDifficultyPassed (TODO: remove after the fork) + OverrideTerminalTotalDifficultyPassed *bool `toml:",omitempty"` } // CreateConsensusEngine creates a consensus engine for the given chain configuration. diff --git a/eth/ethconfig/gen_config.go b/eth/ethconfig/gen_config.go index 0f43c36ad4472..9c7a04364d201 100644 --- a/eth/ethconfig/gen_config.go +++ b/eth/ethconfig/gen_config.go @@ -18,49 +18,50 @@ import ( // MarshalTOML marshals as TOML. func (c Config) MarshalTOML() (interface{}, error) { type Config struct { - Genesis *core.Genesis `toml:",omitempty"` - NetworkId uint64 - SyncMode downloader.SyncMode - EthDiscoveryURLs []string - SnapDiscoveryURLs []string - NoPruning bool - NoPrefetch bool - TxLookupLimit uint64 `toml:",omitempty"` - RequiredBlocks map[uint64]common.Hash `toml:"-"` - LightServ int `toml:",omitempty"` - LightIngress int `toml:",omitempty"` - LightEgress int `toml:",omitempty"` - LightPeers int `toml:",omitempty"` - LightNoPrune bool `toml:",omitempty"` - LightNoSyncServe bool `toml:",omitempty"` - SyncFromCheckpoint bool `toml:",omitempty"` - UltraLightServers []string `toml:",omitempty"` - UltraLightFraction int `toml:",omitempty"` - UltraLightOnlyAnnounce bool `toml:",omitempty"` - SkipBcVersionCheck bool `toml:"-"` - DatabaseHandles int `toml:"-"` - DatabaseCache int - DatabaseFreezer string - TrieCleanCache int - TrieCleanCacheJournal string `toml:",omitempty"` - TrieCleanCacheRejournal time.Duration `toml:",omitempty"` - TrieDirtyCache int - TrieTimeout time.Duration - SnapshotCache int - Preimages bool - Miner miner.Config - Ethash ethash.Config - TxPool core.TxPoolConfig - GPO gasprice.Config - EnablePreimageRecording bool - DocRoot string `toml:"-"` - RPCGasCap uint64 - RPCEVMTimeout time.Duration - RPCTxFeeCap float64 - Checkpoint *params.TrustedCheckpoint `toml:",omitempty"` - CheckpointOracle *params.CheckpointOracleConfig `toml:",omitempty"` - OverrideArrowGlacier *big.Int `toml:",omitempty"` - OverrideTerminalTotalDifficulty *big.Int `toml:",omitempty"` + Genesis *core.Genesis `toml:",omitempty"` + NetworkId uint64 + SyncMode downloader.SyncMode + EthDiscoveryURLs []string + SnapDiscoveryURLs []string + NoPruning bool + NoPrefetch bool + TxLookupLimit uint64 `toml:",omitempty"` + RequiredBlocks map[uint64]common.Hash `toml:"-"` + LightServ int `toml:",omitempty"` + LightIngress int `toml:",omitempty"` + LightEgress int `toml:",omitempty"` + LightPeers int `toml:",omitempty"` + LightNoPrune bool `toml:",omitempty"` + LightNoSyncServe bool `toml:",omitempty"` + SyncFromCheckpoint bool `toml:",omitempty"` + UltraLightServers []string `toml:",omitempty"` + UltraLightFraction int `toml:",omitempty"` + UltraLightOnlyAnnounce bool `toml:",omitempty"` + SkipBcVersionCheck bool `toml:"-"` + DatabaseHandles int `toml:"-"` + DatabaseCache int + DatabaseFreezer string + TrieCleanCache int + TrieCleanCacheJournal string `toml:",omitempty"` + TrieCleanCacheRejournal time.Duration `toml:",omitempty"` + TrieDirtyCache int + TrieTimeout time.Duration + SnapshotCache int + Preimages bool + FilterLogCacheSize int + Miner miner.Config + Ethash ethash.Config + TxPool core.TxPoolConfig + GPO gasprice.Config + EnablePreimageRecording bool + DocRoot string `toml:"-"` + RPCGasCap uint64 + RPCEVMTimeout time.Duration + RPCTxFeeCap float64 + Checkpoint *params.TrustedCheckpoint `toml:",omitempty"` + CheckpointOracle *params.CheckpointOracleConfig `toml:",omitempty"` + OverrideTerminalTotalDifficulty *big.Int `toml:",omitempty"` + OverrideTerminalTotalDifficultyPassed *bool `toml:",omitempty"` } var enc Config enc.Genesis = c.Genesis @@ -93,6 +94,7 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.TrieTimeout = c.TrieTimeout enc.SnapshotCache = c.SnapshotCache enc.Preimages = c.Preimages + enc.FilterLogCacheSize = c.FilterLogCacheSize enc.Miner = c.Miner enc.Ethash = c.Ethash enc.TxPool = c.TxPool @@ -104,57 +106,58 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.RPCTxFeeCap = c.RPCTxFeeCap enc.Checkpoint = c.Checkpoint enc.CheckpointOracle = c.CheckpointOracle - enc.OverrideArrowGlacier = c.OverrideArrowGlacier enc.OverrideTerminalTotalDifficulty = c.OverrideTerminalTotalDifficulty + enc.OverrideTerminalTotalDifficultyPassed = c.OverrideTerminalTotalDifficultyPassed return &enc, nil } // UnmarshalTOML unmarshals from TOML. func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { type Config struct { - Genesis *core.Genesis `toml:",omitempty"` - NetworkId *uint64 - SyncMode *downloader.SyncMode - EthDiscoveryURLs []string - SnapDiscoveryURLs []string - NoPruning *bool - NoPrefetch *bool - TxLookupLimit *uint64 `toml:",omitempty"` - RequiredBlocks map[uint64]common.Hash `toml:"-"` - LightServ *int `toml:",omitempty"` - LightIngress *int `toml:",omitempty"` - LightEgress *int `toml:",omitempty"` - LightPeers *int `toml:",omitempty"` - LightNoPrune *bool `toml:",omitempty"` - LightNoSyncServe *bool `toml:",omitempty"` - SyncFromCheckpoint *bool `toml:",omitempty"` - UltraLightServers []string `toml:",omitempty"` - UltraLightFraction *int `toml:",omitempty"` - UltraLightOnlyAnnounce *bool `toml:",omitempty"` - SkipBcVersionCheck *bool `toml:"-"` - DatabaseHandles *int `toml:"-"` - DatabaseCache *int - DatabaseFreezer *string - TrieCleanCache *int - TrieCleanCacheJournal *string `toml:",omitempty"` - TrieCleanCacheRejournal *time.Duration `toml:",omitempty"` - TrieDirtyCache *int - TrieTimeout *time.Duration - SnapshotCache *int - Preimages *bool - Miner *miner.Config - Ethash *ethash.Config - TxPool *core.TxPoolConfig - GPO *gasprice.Config - EnablePreimageRecording *bool - DocRoot *string `toml:"-"` - RPCGasCap *uint64 - RPCEVMTimeout *time.Duration - RPCTxFeeCap *float64 - Checkpoint *params.TrustedCheckpoint `toml:",omitempty"` - CheckpointOracle *params.CheckpointOracleConfig `toml:",omitempty"` - OverrideArrowGlacier *big.Int `toml:",omitempty"` - OverrideTerminalTotalDifficulty *big.Int `toml:",omitempty"` + Genesis *core.Genesis `toml:",omitempty"` + NetworkId *uint64 + SyncMode *downloader.SyncMode + EthDiscoveryURLs []string + SnapDiscoveryURLs []string + NoPruning *bool + NoPrefetch *bool + TxLookupLimit *uint64 `toml:",omitempty"` + RequiredBlocks map[uint64]common.Hash `toml:"-"` + LightServ *int `toml:",omitempty"` + LightIngress *int `toml:",omitempty"` + LightEgress *int `toml:",omitempty"` + LightPeers *int `toml:",omitempty"` + LightNoPrune *bool `toml:",omitempty"` + LightNoSyncServe *bool `toml:",omitempty"` + SyncFromCheckpoint *bool `toml:",omitempty"` + UltraLightServers []string `toml:",omitempty"` + UltraLightFraction *int `toml:",omitempty"` + UltraLightOnlyAnnounce *bool `toml:",omitempty"` + SkipBcVersionCheck *bool `toml:"-"` + DatabaseHandles *int `toml:"-"` + DatabaseCache *int + DatabaseFreezer *string + TrieCleanCache *int + TrieCleanCacheJournal *string `toml:",omitempty"` + TrieCleanCacheRejournal *time.Duration `toml:",omitempty"` + TrieDirtyCache *int + TrieTimeout *time.Duration + SnapshotCache *int + Preimages *bool + FilterLogCacheSize *int + Miner *miner.Config + Ethash *ethash.Config + TxPool *core.TxPoolConfig + GPO *gasprice.Config + EnablePreimageRecording *bool + DocRoot *string `toml:"-"` + RPCGasCap *uint64 + RPCEVMTimeout *time.Duration + RPCTxFeeCap *float64 + Checkpoint *params.TrustedCheckpoint `toml:",omitempty"` + CheckpointOracle *params.CheckpointOracleConfig `toml:",omitempty"` + OverrideTerminalTotalDifficulty *big.Int `toml:",omitempty"` + OverrideTerminalTotalDifficultyPassed *bool `toml:",omitempty"` } var dec Config if err := unmarshal(&dec); err != nil { @@ -250,6 +253,9 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.Preimages != nil { c.Preimages = *dec.Preimages } + if dec.FilterLogCacheSize != nil { + c.FilterLogCacheSize = *dec.FilterLogCacheSize + } if dec.Miner != nil { c.Miner = *dec.Miner } @@ -283,11 +289,11 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.CheckpointOracle != nil { c.CheckpointOracle = dec.CheckpointOracle } - if dec.OverrideArrowGlacier != nil { - c.OverrideArrowGlacier = dec.OverrideArrowGlacier - } if dec.OverrideTerminalTotalDifficulty != nil { c.OverrideTerminalTotalDifficulty = dec.OverrideTerminalTotalDifficulty } + if dec.OverrideTerminalTotalDifficultyPassed != nil { + c.OverrideTerminalTotalDifficultyPassed = dec.OverrideTerminalTotalDifficultyPassed + } return nil } diff --git a/eth/fetcher/block_fetcher.go b/eth/fetcher/block_fetcher.go index d75ba3f8e0ee0..bd1a34c83c00d 100644 --- a/eth/fetcher/block_fetcher.go +++ b/eth/fetcher/block_fetcher.go @@ -692,7 +692,6 @@ func (f *BlockFetcher) loop() { } else { f.forgetHash(hash) } - } if matched { task.transactions = append(task.transactions[:i], task.transactions[i+1:]...) diff --git a/eth/fetcher/block_fetcher_test.go b/eth/fetcher/block_fetcher_test.go index 06c61ae55d205..bf7946952e498 100644 --- a/eth/fetcher/block_fetcher_test.go +++ b/eth/fetcher/block_fetcher_test.go @@ -36,10 +36,14 @@ import ( ) var ( - testdb = rawdb.NewMemoryDatabase() - testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - testAddress = crypto.PubkeyToAddress(testKey.PublicKey) - genesis = core.GenesisBlockForTesting(testdb, testAddress, big.NewInt(1000000000000000)) + testdb = rawdb.NewMemoryDatabase() + testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + testAddress = crypto.PubkeyToAddress(testKey.PublicKey) + gspec = core.Genesis{ + Alloc: core.GenesisAlloc{testAddress: {Balance: big.NewInt(1000000000000000)}}, + BaseFee: big.NewInt(params.InitialBaseFee), + } + genesis = gspec.MustCommit(testdb) unknownBlock = types.NewBlock(&types.Header{GasLimit: params.GenesisGasLimit, BaseFee: big.NewInt(params.InitialBaseFee)}, nil, nil, nil, trie.NewStackTrie(nil)) ) diff --git a/eth/fetcher/tx_fetcher.go b/eth/fetcher/tx_fetcher.go index a23cd24bf106e..035e0c2ec7d84 100644 --- a/eth/fetcher/tx_fetcher.go +++ b/eth/fetcher/tx_fetcher.go @@ -120,7 +120,7 @@ type txDelivery struct { direct bool // Whether this is a direct reply or a broadcast } -// txDrop is the notiication that a peer has disconnected. +// txDrop is the notification that a peer has disconnected. type txDrop struct { peer string } @@ -260,7 +260,7 @@ func (f *TxFetcher) Notify(peer string, hashes []common.Hash) error { // Enqueue imports a batch of received transaction into the transaction pool // and the fetcher. This method may be called by both transaction broadcasts and // direct request replies. The differentiation is important so the fetcher can -// re-shedule missing transactions as soon as possible. +// re-schedule missing transactions as soon as possible. func (f *TxFetcher) Enqueue(peer string, txs []*types.Transaction, direct bool) error { // Keep track of all the propagated transactions if direct { @@ -558,7 +558,7 @@ func (f *TxFetcher) loop() { // In case of a direct delivery, also reschedule anything missing // from the original query if delivery.direct { - // Mark the reqesting successful (independent of individual status) + // Mark the requesting successful (independent of individual status) txRequestDoneMeter.Mark(int64(len(delivery.hashes))) // Make sure something was pending, nuke it @@ -607,7 +607,7 @@ func (f *TxFetcher) loop() { delete(f.alternates, hash) delete(f.fetching, hash) } - // Something was delivered, try to rechedule requests + // Something was delivered, try to reschedule requests f.scheduleFetches(timeoutTimer, timeoutTrigger, nil) // Partial delivery may enable others to deliver too } @@ -719,7 +719,7 @@ func (f *TxFetcher) rescheduleWait(timer *mclock.Timer, trigger chan struct{}) { // should be rescheduled if some request is pending. In practice, a timeout will // cause the timer to be rescheduled every 5 secs (until the peer comes through or // disconnects). This is a limitation of the fetcher code because we don't trac -// pending requests and timed out requests separatey. Without double tracking, if +// pending requests and timed out requests separately. Without double tracking, if // we simply didn't reschedule the timer on all-timeout then the timer would never // be set again since len(request) > 0 => something's running. func (f *TxFetcher) rescheduleTimeout(timer *mclock.Timer, trigger chan struct{}) { diff --git a/eth/fetcher/tx_fetcher_test.go b/eth/fetcher/tx_fetcher_test.go index ce8d02af7ddff..4c06712f7759d 100644 --- a/eth/fetcher/tx_fetcher_test.go +++ b/eth/fetcher/tx_fetcher_test.go @@ -1011,7 +1011,7 @@ func TestTransactionFetcherOutOfBoundDeliveries(t *testing.T) { } // Tests that dropping a peer cleans out all internal data structures in all the -// live or danglng stages. +// live or dangling stages. func TestTransactionFetcherDrop(t *testing.T) { testTransactionFetcherParallel(t, txFetcherTest{ init: func() *TxFetcher { @@ -1121,7 +1121,7 @@ func TestTransactionFetcherDropRescheduling(t *testing.T) { } // This test reproduces a crash caught by the fuzzer. The root cause was a -// dangling transaction timing out and clashing on readd with a concurrently +// dangling transaction timing out and clashing on re-add with a concurrently // announced one. func TestTransactionFetcherFuzzCrash01(t *testing.T) { testTransactionFetcherParallel(t, txFetcherTest{ @@ -1148,7 +1148,7 @@ func TestTransactionFetcherFuzzCrash01(t *testing.T) { } // This test reproduces a crash caught by the fuzzer. The root cause was a -// dangling transaction getting peer-dropped and clashing on readd with a +// dangling transaction getting peer-dropped and clashing on re-add with a // concurrently announced one. func TestTransactionFetcherFuzzCrash02(t *testing.T) { testTransactionFetcherParallel(t, txFetcherTest{ diff --git a/eth/filters/api.go b/eth/filters/api.go index 7196e90f9eb71..43e63d5ba98a8 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -36,39 +36,39 @@ import ( // and associated subscription in the event system. type filter struct { typ Type - deadline *time.Timer // filter is inactiv when deadline triggers + deadline *time.Timer // filter is inactive when deadline triggers hashes []common.Hash crit FilterCriteria logs []*types.Log s *Subscription // associated subscription in event system } -// PublicFilterAPI offers support to create and manage filters. This will allow external clients to retrieve various +// FilterAPI offers support to create and manage filters. This will allow external clients to retrieve various // information related to the Ethereum protocol such als blocks, transactions and logs. -type PublicFilterAPI struct { - backend Backend +type FilterAPI struct { + sys *FilterSystem events *EventSystem filtersMu sync.Mutex filters map[rpc.ID]*filter timeout time.Duration } -// NewPublicFilterAPI returns a new PublicFilterAPI instance. -func NewPublicFilterAPI(backend Backend, lightMode bool, timeout time.Duration) *PublicFilterAPI { - api := &PublicFilterAPI{ - backend: backend, - events: NewEventSystem(backend, lightMode), +// NewFilterAPI returns a new FilterAPI instance. +func NewFilterAPI(system *FilterSystem, lightMode bool) *FilterAPI { + api := &FilterAPI{ + sys: system, + events: NewEventSystem(system, lightMode), filters: make(map[rpc.ID]*filter), - timeout: timeout, + timeout: system.cfg.Timeout, } - go api.timeoutLoop(timeout) + go api.timeoutLoop(system.cfg.Timeout) return api } // timeoutLoop runs at the interval set by 'timeout' and deletes filters // that have not been recently used. It is started when the API is created. -func (api *PublicFilterAPI) timeoutLoop(timeout time.Duration) { +func (api *FilterAPI) timeoutLoop(timeout time.Duration) { var toUninstall []*Subscription ticker := time.NewTicker(timeout) defer ticker.Stop() @@ -101,9 +101,7 @@ func (api *PublicFilterAPI) timeoutLoop(timeout time.Duration) { // // It is part of the filter package because this filter can be used through the // `eth_getFilterChanges` polling method that is also used for log filters. -// -// https://eth.wiki/json-rpc/API#eth_newpendingtransactionfilter -func (api *PublicFilterAPI) NewPendingTransactionFilter() rpc.ID { +func (api *FilterAPI) NewPendingTransactionFilter() rpc.ID { var ( pendingTxs = make(chan []common.Hash) pendingTxSub = api.events.SubscribePendingTxs(pendingTxs) @@ -136,7 +134,7 @@ func (api *PublicFilterAPI) NewPendingTransactionFilter() rpc.ID { // NewPendingTransactions creates a subscription that is triggered each time a transaction // enters the transaction pool and was signed from one of the transactions this nodes manages. -func (api *PublicFilterAPI) NewPendingTransactions(ctx context.Context) (*rpc.Subscription, error) { +func (api *FilterAPI) NewPendingTransactions(ctx context.Context) (*rpc.Subscription, error) { notifier, supported := rpc.NotifierFromContext(ctx) if !supported { return &rpc.Subscription{}, rpc.ErrNotificationsUnsupported @@ -171,9 +169,7 @@ func (api *PublicFilterAPI) NewPendingTransactions(ctx context.Context) (*rpc.Su // NewBlockFilter creates a filter that fetches blocks that are imported into the chain. // It is part of the filter package since polling goes with eth_getFilterChanges. -// -// https://eth.wiki/json-rpc/API#eth_newblockfilter -func (api *PublicFilterAPI) NewBlockFilter() rpc.ID { +func (api *FilterAPI) NewBlockFilter() rpc.ID { var ( headers = make(chan *types.Header) headerSub = api.events.SubscribeNewHeads(headers) @@ -205,7 +201,7 @@ func (api *PublicFilterAPI) NewBlockFilter() rpc.ID { } // NewHeads send a notification each time a new (header) block is appended to the chain. -func (api *PublicFilterAPI) NewHeads(ctx context.Context) (*rpc.Subscription, error) { +func (api *FilterAPI) NewHeads(ctx context.Context) (*rpc.Subscription, error) { notifier, supported := rpc.NotifierFromContext(ctx) if !supported { return &rpc.Subscription{}, rpc.ErrNotificationsUnsupported @@ -235,7 +231,7 @@ func (api *PublicFilterAPI) NewHeads(ctx context.Context) (*rpc.Subscription, er } // Logs creates a subscription that fires for all new log that match the given filter criteria. -func (api *PublicFilterAPI) Logs(ctx context.Context, crit FilterCriteria) (*rpc.Subscription, error) { +func (api *FilterAPI) Logs(ctx context.Context, crit FilterCriteria) (*rpc.Subscription, error) { notifier, supported := rpc.NotifierFromContext(ctx) if !supported { return &rpc.Subscription{}, rpc.ErrNotificationsUnsupported @@ -252,11 +248,11 @@ func (api *PublicFilterAPI) Logs(ctx context.Context, crit FilterCriteria) (*rpc } go func() { - for { select { case logs := <-matchedLogs: for _, log := range logs { + log := log notifier.Notify(rpcSub.ID, &log) } case <-rpcSub.Err(): // client send an unsubscribe request @@ -287,9 +283,7 @@ type FilterCriteria ethereum.FilterQuery // again but with the removed property set to true. // // In case "fromBlock" > "toBlock" an error is returned. -// -// https://eth.wiki/json-rpc/API#eth_newfilter -func (api *PublicFilterAPI) NewFilter(crit FilterCriteria) (rpc.ID, error) { +func (api *FilterAPI) NewFilter(crit FilterCriteria) (rpc.ID, error) { logs := make(chan []*types.Log) logsSub, err := api.events.SubscribeLogs(ethereum.FilterQuery(crit), logs) if err != nil { @@ -322,13 +316,11 @@ func (api *PublicFilterAPI) NewFilter(crit FilterCriteria) (rpc.ID, error) { } // GetLogs returns logs matching the given argument that are stored within the state. -// -// https://eth.wiki/json-rpc/API#eth_getlogs -func (api *PublicFilterAPI) GetLogs(ctx context.Context, crit FilterCriteria) ([]*types.Log, error) { +func (api *FilterAPI) GetLogs(ctx context.Context, crit FilterCriteria) ([]*types.Log, error) { var filter *Filter if crit.BlockHash != nil { // Block filter requested, construct a single-shot filter - filter = NewBlockFilter(api.backend, *crit.BlockHash, crit.Addresses, crit.Topics) + filter = api.sys.NewBlockFilter(*crit.BlockHash, crit.Addresses, crit.Topics) } else { // Convert the RPC block numbers into internal representations begin := rpc.LatestBlockNumber.Int64() @@ -340,7 +332,7 @@ func (api *PublicFilterAPI) GetLogs(ctx context.Context, crit FilterCriteria) ([ end = crit.ToBlock.Int64() } // Construct the range filter - filter = NewRangeFilter(api.backend, begin, end, crit.Addresses, crit.Topics) + filter = api.sys.NewRangeFilter(begin, end, crit.Addresses, crit.Topics) } // Run the filter and return all the logs logs, err := filter.Logs(ctx) @@ -351,9 +343,7 @@ func (api *PublicFilterAPI) GetLogs(ctx context.Context, crit FilterCriteria) ([ } // UninstallFilter removes the filter with the given filter id. -// -// https://eth.wiki/json-rpc/API#eth_uninstallfilter -func (api *PublicFilterAPI) UninstallFilter(id rpc.ID) bool { +func (api *FilterAPI) UninstallFilter(id rpc.ID) bool { api.filtersMu.Lock() f, found := api.filters[id] if found { @@ -369,9 +359,7 @@ func (api *PublicFilterAPI) UninstallFilter(id rpc.ID) bool { // GetFilterLogs returns the logs for the filter with the given id. // If the filter could not be found an empty array of logs is returned. -// -// https://eth.wiki/json-rpc/API#eth_getfilterlogs -func (api *PublicFilterAPI) GetFilterLogs(ctx context.Context, id rpc.ID) ([]*types.Log, error) { +func (api *FilterAPI) GetFilterLogs(ctx context.Context, id rpc.ID) ([]*types.Log, error) { api.filtersMu.Lock() f, found := api.filters[id] api.filtersMu.Unlock() @@ -383,7 +371,7 @@ func (api *PublicFilterAPI) GetFilterLogs(ctx context.Context, id rpc.ID) ([]*ty var filter *Filter if f.crit.BlockHash != nil { // Block filter requested, construct a single-shot filter - filter = NewBlockFilter(api.backend, *f.crit.BlockHash, f.crit.Addresses, f.crit.Topics) + filter = api.sys.NewBlockFilter(*f.crit.BlockHash, f.crit.Addresses, f.crit.Topics) } else { // Convert the RPC block numbers into internal representations begin := rpc.LatestBlockNumber.Int64() @@ -395,7 +383,7 @@ func (api *PublicFilterAPI) GetFilterLogs(ctx context.Context, id rpc.ID) ([]*ty end = f.crit.ToBlock.Int64() } // Construct the range filter - filter = NewRangeFilter(api.backend, begin, end, f.crit.Addresses, f.crit.Topics) + filter = api.sys.NewRangeFilter(begin, end, f.crit.Addresses, f.crit.Topics) } // Run the filter and return all the logs logs, err := filter.Logs(ctx) @@ -410,9 +398,7 @@ func (api *PublicFilterAPI) GetFilterLogs(ctx context.Context, id rpc.ID) ([]*ty // // For pending transaction and block filters the result is []common.Hash. // (pending)Log filters return []Log. -// -// https://eth.wiki/json-rpc/API#eth_getfilterchanges -func (api *PublicFilterAPI) GetFilterChanges(id rpc.ID) (interface{}, error) { +func (api *FilterAPI) GetFilterChanges(id rpc.ID) (interface{}, error) { api.filtersMu.Lock() defer api.filtersMu.Unlock() diff --git a/eth/filters/api_test.go b/eth/filters/api_test.go index 02229a7549a78..0a80d0f8ddbde 100644 --- a/eth/filters/api_test.go +++ b/eth/filters/api_test.go @@ -56,7 +56,7 @@ func TestUnmarshalJSONNewFilterArgs(t *testing.T) { // from, to block number var test1 FilterCriteria - vector := fmt.Sprintf(`{"fromBlock":"0x%x","toBlock":"0x%x"}`, fromBlock, toBlock) + vector := fmt.Sprintf(`{"fromBlock":"%#x","toBlock":"%#x"}`, fromBlock, toBlock) if err := json.Unmarshal([]byte(vector), &test1); err != nil { t.Fatal(err) } diff --git a/eth/filters/bench_test.go b/eth/filters/bench_test.go index 9632f4195f4cf..73b96b77af62d 100644 --- a/eth/filters/bench_test.go +++ b/eth/filters/bench_test.go @@ -93,9 +93,9 @@ func benchmarkBloomBits(b *testing.B, sectionSize uint64) { var header *types.Header for i := sectionIdx * sectionSize; i < (sectionIdx+1)*sectionSize; i++ { hash := rawdb.ReadCanonicalHash(db, i) - header = rawdb.ReadHeader(db, hash, i) - if header == nil { + if header = rawdb.ReadHeader(db, hash, i); header == nil { b.Fatalf("Error creating bloomBits data") + return } bc.AddBloom(uint(i-sectionIdx*sectionSize), header.Bloom) } @@ -122,31 +122,36 @@ func benchmarkBloomBits(b *testing.B, sectionSize uint64) { b.Log("Running filter benchmarks...") start = time.Now() - var backend *testBackend + var ( + backend *testBackend + sys *FilterSystem + ) for i := 0; i < benchFilterCnt; i++ { if i%20 == 0 { db.Close() db, _ = rawdb.NewLevelDBDatabase(benchDataDir, 128, 1024, "", false) backend = &testBackend{db: db, sections: cnt} + sys = NewFilterSystem(backend, Config{}) } var addr common.Address addr[0] = byte(i) addr[1] = byte(i / 256) - filter := NewRangeFilter(backend, 0, int64(cnt*sectionSize-1), []common.Address{addr}, nil) + filter := sys.NewRangeFilter(0, int64(cnt*sectionSize-1), []common.Address{addr}, nil) if _, err := filter.Logs(context.Background()); err != nil { - b.Error("filter.Find error:", err) + b.Error("filter.Logs error:", err) } } + d = time.Since(start) b.Log("Finished running filter benchmarks") b.Log(" ", d, "total ", d/time.Duration(benchFilterCnt), "per address", d*time.Duration(1000000)/time.Duration(benchFilterCnt*cnt*sectionSize), "per million blocks") db.Close() } -var bloomBitsPrefix = []byte("bloomBits-") - +//nolint:unused func clearBloomBits(db ethdb.Database) { + var bloomBitsPrefix = []byte("bloomBits-") fmt.Println("Clearing bloombits data...") it := db.NewIterator(bloomBitsPrefix, nil) for it.Next() { @@ -171,10 +176,11 @@ func BenchmarkNoBloomBits(b *testing.B) { clearBloomBits(db) + _, sys := newTestFilterSystem(b, db, Config{}) + b.Log("Running filter benchmarks...") start := time.Now() - backend := &testBackend{db: db} - filter := NewRangeFilter(backend, 0, int64(*headNum), []common.Address{{}}, nil) + filter := sys.NewRangeFilter(0, int64(*headNum), []common.Address{{}}, nil) filter.Logs(context.Background()) d := time.Since(start) b.Log("Finished running filter benchmarks") diff --git a/eth/filters/filter.go b/eth/filters/filter.go index f64e84abb86c9..8d80647fc241f 100644 --- a/eth/filters/filter.go +++ b/eth/filters/filter.go @@ -22,48 +22,27 @@ import ( "math/big" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/bloombits" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/rpc" ) -type Backend interface { - ChainDb() ethdb.Database - HeaderByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*types.Header, error) - HeaderByHash(ctx context.Context, blockHash common.Hash) (*types.Header, error) - GetReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error) - GetLogs(ctx context.Context, blockHash common.Hash) ([][]*types.Log, error) - - SubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscription - SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription - SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription - SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription - SubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription - - BloomStatus() (uint64, uint64) - ServiceFilter(ctx context.Context, session *bloombits.MatcherSession) -} - // Filter can be used to retrieve and filter logs. type Filter struct { - backend Backend + sys *FilterSystem - db ethdb.Database addresses []common.Address topics [][]common.Hash - block common.Hash // Block hash if filtering a single block - begin, end int64 // Range interval if filtering multiple blocks + block *common.Hash // Block hash if filtering a single block + begin, end int64 // Range interval if filtering multiple blocks matcher *bloombits.Matcher } // NewRangeFilter creates a new filter which uses a bloom filter on blocks to // figure out whether a particular block is interesting or not. -func NewRangeFilter(backend Backend, begin, end int64, addresses []common.Address, topics [][]common.Hash) *Filter { +func (sys *FilterSystem) NewRangeFilter(begin, end int64, addresses []common.Address, topics [][]common.Hash) *Filter { // Flatten the address and topic filter clauses into a single bloombits filter // system. Since the bloombits are not positional, nil topics are permitted, // which get flattened into a nil byte slice. @@ -82,10 +61,10 @@ func NewRangeFilter(backend Backend, begin, end int64, addresses []common.Addres } filters = append(filters, filter) } - size, _ := backend.BloomStatus() + size, _ := sys.backend.BloomStatus() // Create a generic filter and convert it into a range filter - filter := newFilter(backend, addresses, topics) + filter := newFilter(sys, addresses, topics) filter.matcher = bloombits.NewMatcher(size, filters) filter.begin = begin @@ -96,21 +75,20 @@ func NewRangeFilter(backend Backend, begin, end int64, addresses []common.Addres // NewBlockFilter creates a new filter which directly inspects the contents of // a block to figure out whether it is interesting or not. -func NewBlockFilter(backend Backend, block common.Hash, addresses []common.Address, topics [][]common.Hash) *Filter { +func (sys *FilterSystem) NewBlockFilter(block common.Hash, addresses []common.Address, topics [][]common.Hash) *Filter { // Create a generic filter and convert it into a block filter - filter := newFilter(backend, addresses, topics) - filter.block = block + filter := newFilter(sys, addresses, topics) + filter.block = &block return filter } // newFilter creates a generic filter that can either filter based on a block hash, // or based on range queries. The search criteria needs to be explicitly set. -func newFilter(backend Backend, addresses []common.Address, topics [][]common.Hash) *Filter { +func newFilter(sys *FilterSystem, addresses []common.Address, topics [][]common.Hash) *Filter { return &Filter{ - backend: backend, + sys: sys, addresses: addresses, topics: topics, - db: backend.ChainDb(), } } @@ -118,36 +96,45 @@ func newFilter(backend Backend, addresses []common.Address, topics [][]common.Ha // first block that contains matches, updating the start of the filter accordingly. func (f *Filter) Logs(ctx context.Context) ([]*types.Log, error) { // If we're doing singleton block filtering, execute and return - if f.block != (common.Hash{}) { - header, err := f.backend.HeaderByHash(ctx, f.block) + if f.block != nil { + header, err := f.sys.backend.HeaderByHash(ctx, *f.block) if err != nil { return nil, err } if header == nil { return nil, errors.New("unknown block") } - return f.blockLogs(ctx, header) + return f.blockLogs(ctx, header, false) + } + // Short-cut if all we care about is pending logs + if f.begin == rpc.PendingBlockNumber.Int64() { + if f.end != rpc.PendingBlockNumber.Int64() { + return nil, errors.New("invalid block range") + } + return f.pendingLogs() } // Figure out the limits of the filter range - header, _ := f.backend.HeaderByNumber(ctx, rpc.LatestBlockNumber) + header, _ := f.sys.backend.HeaderByNumber(ctx, rpc.LatestBlockNumber) if header == nil { return nil, nil } - head := header.Number.Uint64() - - if f.begin == -1 { + var ( + head = header.Number.Uint64() + end = uint64(f.end) + pending = f.end == rpc.PendingBlockNumber.Int64() + ) + if f.begin == rpc.LatestBlockNumber.Int64() { f.begin = int64(head) } - end := uint64(f.end) - if f.end == -1 { + if f.end == rpc.LatestBlockNumber.Int64() || f.end == rpc.PendingBlockNumber.Int64() { end = head } // Gather all indexed logs, and finish with non indexed ones var ( - logs []*types.Log - err error + logs []*types.Log + err error + size, sections = f.sys.backend.BloomStatus() ) - size, sections := f.backend.BloomStatus() if indexed := sections * size; indexed > uint64(f.begin) { if indexed > end { logs, err = f.indexedLogs(ctx, end) @@ -160,6 +147,13 @@ func (f *Filter) Logs(ctx context.Context) ([]*types.Log, error) { } rest, err := f.unindexedLogs(ctx, end) logs = append(logs, rest...) + if pending { + pendingLogs, err := f.pendingLogs() + if err != nil { + return nil, err + } + logs = append(logs, pendingLogs...) + } return logs, err } @@ -175,7 +169,7 @@ func (f *Filter) indexedLogs(ctx context.Context, end uint64) ([]*types.Log, err } defer session.Close() - f.backend.ServiceFilter(ctx, session) + f.sys.backend.ServiceFilter(ctx, session) // Iterate over the matches until exhausted or context closed var logs []*types.Log @@ -194,11 +188,11 @@ func (f *Filter) indexedLogs(ctx context.Context, end uint64) ([]*types.Log, err f.begin = int64(number) + 1 // Retrieve the suggested block and pull any truly matching logs - header, err := f.backend.HeaderByNumber(ctx, rpc.BlockNumber(number)) + header, err := f.sys.backend.HeaderByNumber(ctx, rpc.BlockNumber(number)) if header == nil || err != nil { return logs, err } - found, err := f.checkMatches(ctx, header) + found, err := f.blockLogs(ctx, header, true) if err != nil { return logs, err } @@ -216,11 +210,11 @@ func (f *Filter) unindexedLogs(ctx context.Context, end uint64) ([]*types.Log, e var logs []*types.Log for ; f.begin <= int64(end); f.begin++ { - header, err := f.backend.HeaderByNumber(ctx, rpc.BlockNumber(f.begin)) + header, err := f.sys.backend.HeaderByNumber(ctx, rpc.BlockNumber(f.begin)) if header == nil || err != nil { return logs, err } - found, err := f.blockLogs(ctx, header) + found, err := f.blockLogs(ctx, header, false) if err != nil { return logs, err } @@ -230,34 +224,34 @@ func (f *Filter) unindexedLogs(ctx context.Context, end uint64) ([]*types.Log, e } // blockLogs returns the logs matching the filter criteria within a single block. -func (f *Filter) blockLogs(ctx context.Context, header *types.Header) (logs []*types.Log, err error) { - if bloomFilter(header.Bloom, f.addresses, f.topics) { - found, err := f.checkMatches(ctx, header) +func (f *Filter) blockLogs(ctx context.Context, header *types.Header, skipBloom bool) ([]*types.Log, error) { + // Fast track: no filtering criteria + if len(f.addresses) == 0 && len(f.topics) == 0 { + list, err := f.sys.cachedGetLogs(ctx, header.Hash(), header.Number.Uint64()) if err != nil { - return logs, err + return nil, err } - logs = append(logs, found...) + return flatten(list), nil + } else if skipBloom || bloomFilter(header.Bloom, f.addresses, f.topics) { + return f.checkMatches(ctx, header) } - return logs, nil + return nil, nil } // checkMatches checks if the receipts belonging to the given header contain any log events that // match the filter criteria. This function is called when the bloom filter signals a potential match. -func (f *Filter) checkMatches(ctx context.Context, header *types.Header) (logs []*types.Log, err error) { - // Get the logs of the block - logsList, err := f.backend.GetLogs(ctx, header.Hash()) +func (f *Filter) checkMatches(ctx context.Context, header *types.Header) ([]*types.Log, error) { + logsList, err := f.sys.cachedGetLogs(ctx, header.Hash(), header.Number.Uint64()) if err != nil { return nil, err } - var unfiltered []*types.Log - for _, logs := range logsList { - unfiltered = append(unfiltered, logs...) - } - logs = filterLogs(unfiltered, nil, nil, f.addresses, f.topics) + + unfiltered := flatten(logsList) + logs := filterLogs(unfiltered, nil, nil, f.addresses, f.topics) if len(logs) > 0 { // We have matching logs, check if we need to resolve full logs via the light client if logs[0].TxHash == (common.Hash{}) { - receipts, err := f.backend.GetReceipts(ctx, header.Hash()) + receipts, err := f.sys.backend.GetReceipts(ctx, header.Hash()) if err != nil { return nil, err } @@ -272,6 +266,19 @@ func (f *Filter) checkMatches(ctx context.Context, header *types.Header) (logs [ return nil, nil } +// pendingLogs returns the logs matching the filter criteria within the pending block. +func (f *Filter) pendingLogs() ([]*types.Log, error) { + block, receipts := f.sys.backend.PendingBlockAndReceipts() + if bloomFilter(block.Bloom(), f.addresses, f.topics) { + var unfiltered []*types.Log + for _, r := range receipts { + unfiltered = append(unfiltered, r.Logs...) + } + return filterLogs(unfiltered, nil, nil, f.addresses, f.topics), nil + } + return nil, nil +} + func includes(addresses []common.Address, a common.Address) bool { for _, addr := range addresses { if addr == a { @@ -346,3 +353,11 @@ func bloomFilter(bloom types.Bloom, addresses []common.Address, topics [][]commo } return true } + +func flatten(list [][]*types.Log) []*types.Log { + var flat []*types.Log + for _, logs := range list { + flat = append(flat, logs...) + } + return flat +} diff --git a/eth/filters/filter_system.go b/eth/filters/filter_system.go index c1a1b408b7a74..79a9b089f4224 100644 --- a/eth/filters/filter_system.go +++ b/eth/filters/filter_system.go @@ -27,13 +27,90 @@ import ( "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/bloombits" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rpc" + lru "github.com/hashicorp/golang-lru" ) +// Config represents the configuration of the filter system. +type Config struct { + LogCacheSize int // maximum number of cached blocks (default: 32) + Timeout time.Duration // how long filters stay active (default: 5min) +} + +func (cfg Config) withDefaults() Config { + if cfg.Timeout == 0 { + cfg.Timeout = 5 * time.Minute + } + if cfg.LogCacheSize == 0 { + cfg.LogCacheSize = 32 + } + return cfg +} + +type Backend interface { + ChainDb() ethdb.Database + HeaderByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*types.Header, error) + HeaderByHash(ctx context.Context, blockHash common.Hash) (*types.Header, error) + GetReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error) + GetLogs(ctx context.Context, blockHash common.Hash, number uint64) ([][]*types.Log, error) + PendingBlockAndReceipts() (*types.Block, types.Receipts) + + SubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscription + SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription + SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription + SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription + SubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription + + BloomStatus() (uint64, uint64) + ServiceFilter(ctx context.Context, session *bloombits.MatcherSession) +} + +// FilterSystem holds resources shared by all filters. +type FilterSystem struct { + backend Backend + logsCache *lru.Cache + cfg *Config +} + +// NewFilterSystem creates a filter system. +func NewFilterSystem(backend Backend, config Config) *FilterSystem { + config = config.withDefaults() + + cache, err := lru.New(config.LogCacheSize) + if err != nil { + panic(err) + } + return &FilterSystem{ + backend: backend, + logsCache: cache, + cfg: &config, + } +} + +// cachedGetLogs loads block logs from the backend and caches the result. +func (sys *FilterSystem) cachedGetLogs(ctx context.Context, blockHash common.Hash, number uint64) ([][]*types.Log, error) { + cached, ok := sys.logsCache.Get(blockHash) + if ok { + return cached.([][]*types.Log), nil + } + + logs, err := sys.backend.GetLogs(ctx, blockHash, number) + if err != nil { + return nil, err + } + if logs == nil { + return nil, fmt.Errorf("failed to get logs for block #%d (0x%s)", number, blockHash.TerminalString()) + } + sys.logsCache.Add(blockHash, logs) + return logs, nil +} + // Type determines the kind of filter and is used to put the filter in to // the correct bucket when added. type Type byte @@ -84,6 +161,7 @@ type subscription struct { // subscription which match the subscription criteria. type EventSystem struct { backend Backend + sys *FilterSystem lightMode bool lastHead *types.Header @@ -110,9 +188,10 @@ type EventSystem struct { // // The returned manager has a loop that needs to be stopped with the Stop function // or by stopping the given mux. -func NewEventSystem(backend Backend, lightMode bool) *EventSystem { +func NewEventSystem(sys *FilterSystem, lightMode bool) *EventSystem { m := &EventSystem{ - backend: backend, + sys: sys, + backend: sys.backend, lightMode: lightMode, install: make(chan *subscription), uninstall: make(chan *subscription), @@ -405,7 +484,7 @@ func (es *EventSystem) lightFilterLogs(header *types.Header, addresses []common. // Get the logs of the block ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) defer cancel() - logsList, err := es.backend.GetLogs(ctx, header.Hash()) + logsList, err := es.sys.cachedGetLogs(ctx, header.Hash(), header.Number.Uint64()) if err != nil { return nil } diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go index 87971d5a97c94..51bda29b42444 100644 --- a/eth/filters/filter_system_test.go +++ b/eth/filters/filter_system_test.go @@ -39,12 +39,7 @@ import ( "github.com/ethereum/go-ethereum/rpc" ) -var ( - deadline = 5 * time.Minute -) - type testBackend struct { - mux *event.TypeMux db ethdb.Database sections uint64 txFeed event.Feed @@ -92,20 +87,15 @@ func (b *testBackend) GetReceipts(ctx context.Context, hash common.Hash) (types. return nil, nil } -func (b *testBackend) GetLogs(ctx context.Context, hash common.Hash) ([][]*types.Log, error) { - number := rawdb.ReadHeaderNumber(b.db, hash) - if number == nil { - return nil, nil - } - receipts := rawdb.ReadReceipts(b.db, hash, *number, params.TestChainConfig) - - logs := make([][]*types.Log, len(receipts)) - for i, receipt := range receipts { - logs[i] = receipt.Logs - } +func (b *testBackend) GetLogs(ctx context.Context, hash common.Hash, number uint64) ([][]*types.Log, error) { + logs := rawdb.ReadLogs(b.db, hash, number, params.TestChainConfig) return logs, nil } +func (b *testBackend) PendingBlockAndReceipts() (*types.Block, types.Receipts) { + return nil, nil +} + func (b *testBackend) SubscribeNewTxsEvent(ch chan<- core.NewTxsEvent) event.Subscription { return b.txFeed.Subscribe(ch) } @@ -157,6 +147,12 @@ func (b *testBackend) ServiceFilter(ctx context.Context, session *bloombits.Matc }() } +func newTestFilterSystem(t testing.TB, db ethdb.Database, cfg Config) (*testBackend, *FilterSystem) { + backend := &testBackend{db: db} + sys := NewFilterSystem(backend, cfg) + return backend, sys +} + // TestBlockSubscription tests if a block subscription returns block hashes for posted chain events. // It creates multiple subscriptions: // - one at the start and should receive all posted chain events and a second (blockHashes) @@ -166,12 +162,12 @@ func TestBlockSubscription(t *testing.T) { t.Parallel() var ( - db = rawdb.NewMemoryDatabase() - backend = &testBackend{db: db} - api = NewPublicFilterAPI(backend, false, deadline) - genesis = (&core.Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(db) - chain, _ = core.GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), db, 10, func(i int, gen *core.BlockGen) {}) - chainEvents = []core.ChainEvent{} + db = rawdb.NewMemoryDatabase() + backend, sys = newTestFilterSystem(t, db, Config{}) + api = NewFilterAPI(sys, false) + genesis = (&core.Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(db) + chain, _ = core.GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), db, 10, func(i int, gen *core.BlockGen) {}) + chainEvents = []core.ChainEvent{} ) for _, blk := range chain { @@ -218,9 +214,9 @@ func TestPendingTxFilter(t *testing.T) { t.Parallel() var ( - db = rawdb.NewMemoryDatabase() - backend = &testBackend{db: db} - api = NewPublicFilterAPI(backend, false, deadline) + db = rawdb.NewMemoryDatabase() + backend, sys = newTestFilterSystem(t, db, Config{}) + api = NewFilterAPI(sys, false) transactions = []*types.Transaction{ types.NewTransaction(0, common.HexToAddress("0xb794f5ea0ba39494ce83a213fffba74279579268"), new(big.Int), 0, new(big.Int), nil), @@ -273,9 +269,9 @@ func TestPendingTxFilter(t *testing.T) { // If not it must return an error. func TestLogFilterCreation(t *testing.T) { var ( - db = rawdb.NewMemoryDatabase() - backend = &testBackend{db: db} - api = NewPublicFilterAPI(backend, false, deadline) + db = rawdb.NewMemoryDatabase() + _, sys = newTestFilterSystem(t, db, Config{}) + api = NewFilterAPI(sys, false) testCases = []struct { crit FilterCriteria @@ -320,9 +316,9 @@ func TestInvalidLogFilterCreation(t *testing.T) { t.Parallel() var ( - db = rawdb.NewMemoryDatabase() - backend = &testBackend{db: db} - api = NewPublicFilterAPI(backend, false, deadline) + db = rawdb.NewMemoryDatabase() + _, sys = newTestFilterSystem(t, db, Config{}) + api = NewFilterAPI(sys, false) ) // different situations where log filter creation should fail. @@ -343,8 +339,8 @@ func TestInvalidLogFilterCreation(t *testing.T) { func TestInvalidGetLogsRequest(t *testing.T) { var ( db = rawdb.NewMemoryDatabase() - backend = &testBackend{db: db} - api = NewPublicFilterAPI(backend, false, deadline) + _, sys = newTestFilterSystem(t, db, Config{}) + api = NewFilterAPI(sys, false) blockHash = common.HexToHash("0x1111111111111111111111111111111111111111111111111111111111111111") ) @@ -367,9 +363,9 @@ func TestLogFilter(t *testing.T) { t.Parallel() var ( - db = rawdb.NewMemoryDatabase() - backend = &testBackend{db: db} - api = NewPublicFilterAPI(backend, false, deadline) + db = rawdb.NewMemoryDatabase() + backend, sys = newTestFilterSystem(t, db, Config{}) + api = NewFilterAPI(sys, false) firstAddr = common.HexToAddress("0x1111111111111111111111111111111111111111") secondAddr = common.HexToAddress("0x2222222222222222222222222222222222222222") @@ -481,9 +477,9 @@ func TestPendingLogsSubscription(t *testing.T) { t.Parallel() var ( - db = rawdb.NewMemoryDatabase() - backend = &testBackend{db: db} - api = NewPublicFilterAPI(backend, false, deadline) + db = rawdb.NewMemoryDatabase() + backend, sys = newTestFilterSystem(t, db, Config{}) + api = NewFilterAPI(sys, false) firstAddr = common.HexToAddress("0x1111111111111111111111111111111111111111") secondAddr = common.HexToAddress("0x2222222222222222222222222222222222222222") @@ -592,7 +588,7 @@ func TestPendingLogsSubscription(t *testing.T) { // (some) events are posted. for i := range testCases { testCases[i].c = make(chan []*types.Log) - testCases[i].err = make(chan error) + testCases[i].err = make(chan error, 1) var err error testCases[i].sub, err = api.events.SubscribeLogs(testCases[i].crit, testCases[i].c) @@ -665,10 +661,10 @@ func TestPendingTxFilterDeadlock(t *testing.T) { timeout := 100 * time.Millisecond var ( - db = rawdb.NewMemoryDatabase() - backend = &testBackend{db: db} - api = NewPublicFilterAPI(backend, false, timeout) - done = make(chan struct{}) + db = rawdb.NewMemoryDatabase() + backend, sys = newTestFilterSystem(t, db, Config{Timeout: timeout}) + api = NewFilterAPI(sys, false) + done = make(chan struct{}) ) go func() { diff --git a/eth/filters/filter_test.go b/eth/filters/filter_test.go index f415046a82aab..2c1f7cadf43a8 100644 --- a/eth/filters/filter_test.go +++ b/eth/filters/filter_test.go @@ -44,16 +44,23 @@ func BenchmarkFilters(b *testing.B) { var ( db, _ = rawdb.NewLevelDBDatabase(dir, 0, 0, "", false) - backend = &testBackend{db: db} + _, sys = newTestFilterSystem(b, db, Config{}) key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") addr1 = crypto.PubkeyToAddress(key1.PublicKey) addr2 = common.BytesToAddress([]byte("jeff")) addr3 = common.BytesToAddress([]byte("ethereum")) addr4 = common.BytesToAddress([]byte("random addresses please")) + + gspec = core.Genesis{ + Alloc: core.GenesisAlloc{addr1: {Balance: big.NewInt(1000000)}}, + BaseFee: big.NewInt(params.InitialBaseFee), + } + genesis = gspec.ToBlock() ) defer db.Close() - genesis := core.GenesisBlockForTesting(db, addr1, big.NewInt(1000000)) + gspec.MustCommit(db) + chain, receipts := core.GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), db, 100010, func(i int, gen *core.BlockGen) { switch i { case 2403: @@ -72,7 +79,6 @@ func BenchmarkFilters(b *testing.B) { receipt := makeReceipt(addr4) gen.AddUncheckedReceipt(receipt) gen.AddUncheckedTx(types.NewTransaction(999, common.HexToAddress("0x999"), big.NewInt(999), 999, gen.BaseFee(), nil)) - } }) for i, block := range chain { @@ -83,7 +89,7 @@ func BenchmarkFilters(b *testing.B) { } b.ResetTimer() - filter := NewRangeFilter(backend, 0, -1, []common.Address{addr1, addr2, addr3, addr4}, nil) + filter := sys.NewRangeFilter(0, -1, []common.Address{addr1, addr2, addr3, addr4}, nil) for i := 0; i < b.N; i++ { logs, _ := filter.Logs(context.Background()) @@ -98,7 +104,7 @@ func TestFilters(t *testing.T) { var ( db, _ = rawdb.NewLevelDBDatabase(dir, 0, 0, "", false) - backend = &testBackend{db: db} + _, sys = newTestFilterSystem(t, db, Config{}) key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") addr = crypto.PubkeyToAddress(key1.PublicKey) @@ -106,10 +112,17 @@ func TestFilters(t *testing.T) { hash2 = common.BytesToHash([]byte("topic2")) hash3 = common.BytesToHash([]byte("topic3")) hash4 = common.BytesToHash([]byte("topic4")) + + gspec = core.Genesis{ + Alloc: core.GenesisAlloc{addr: {Balance: big.NewInt(1000000)}}, + BaseFee: big.NewInt(params.InitialBaseFee), + } + genesis = gspec.ToBlock() ) defer db.Close() - genesis := core.GenesisBlockForTesting(db, addr, big.NewInt(1000000)) + gspec.MustCommit(db) + chain, receipts := core.GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), db, 1000, func(i int, gen *core.BlockGen) { switch i { case 1: @@ -162,14 +175,14 @@ func TestFilters(t *testing.T) { rawdb.WriteReceipts(db, block.Hash(), block.NumberU64(), receipts[i]) } - filter := NewRangeFilter(backend, 0, -1, []common.Address{addr}, [][]common.Hash{{hash1, hash2, hash3, hash4}}) + filter := sys.NewRangeFilter(0, -1, []common.Address{addr}, [][]common.Hash{{hash1, hash2, hash3, hash4}}) logs, _ := filter.Logs(context.Background()) if len(logs) != 4 { t.Error("expected 4 log, got", len(logs)) } - filter = NewRangeFilter(backend, 900, 999, []common.Address{addr}, [][]common.Hash{{hash3}}) + filter = sys.NewRangeFilter(900, 999, []common.Address{addr}, [][]common.Hash{{hash3}}) logs, _ = filter.Logs(context.Background()) if len(logs) != 1 { t.Error("expected 1 log, got", len(logs)) @@ -178,7 +191,7 @@ func TestFilters(t *testing.T) { t.Errorf("expected log[0].Topics[0] to be %x, got %x", hash3, logs[0].Topics[0]) } - filter = NewRangeFilter(backend, 990, -1, []common.Address{addr}, [][]common.Hash{{hash3}}) + filter = sys.NewRangeFilter(990, -1, []common.Address{addr}, [][]common.Hash{{hash3}}) logs, _ = filter.Logs(context.Background()) if len(logs) != 1 { t.Error("expected 1 log, got", len(logs)) @@ -187,7 +200,7 @@ func TestFilters(t *testing.T) { t.Errorf("expected log[0].Topics[0] to be %x, got %x", hash3, logs[0].Topics[0]) } - filter = NewRangeFilter(backend, 1, 10, nil, [][]common.Hash{{hash1, hash2}}) + filter = sys.NewRangeFilter(1, 10, nil, [][]common.Hash{{hash1, hash2}}) logs, _ = filter.Logs(context.Background()) if len(logs) != 2 { @@ -195,7 +208,7 @@ func TestFilters(t *testing.T) { } failHash := common.BytesToHash([]byte("fail")) - filter = NewRangeFilter(backend, 0, -1, nil, [][]common.Hash{{failHash}}) + filter = sys.NewRangeFilter(0, -1, nil, [][]common.Hash{{failHash}}) logs, _ = filter.Logs(context.Background()) if len(logs) != 0 { @@ -203,14 +216,14 @@ func TestFilters(t *testing.T) { } failAddr := common.BytesToAddress([]byte("failmenow")) - filter = NewRangeFilter(backend, 0, -1, []common.Address{failAddr}, nil) + filter = sys.NewRangeFilter(0, -1, []common.Address{failAddr}, nil) logs, _ = filter.Logs(context.Background()) if len(logs) != 0 { t.Error("expected 0 log, got", len(logs)) } - filter = NewRangeFilter(backend, 0, -1, nil, [][]common.Hash{{failHash}, {hash1}}) + filter = sys.NewRangeFilter(0, -1, nil, [][]common.Hash{{failHash}, {hash1}}) logs, _ = filter.Logs(context.Background()) if len(logs) != 0 { diff --git a/eth/gasprice/feehistory.go b/eth/gasprice/feehistory.go index 4113089afb1eb..91835c1641062 100644 --- a/eth/gasprice/feehistory.go +++ b/eth/gasprice/feehistory.go @@ -137,44 +137,68 @@ func (oracle *Oracle) processBlock(bf *blockFees, percentiles []float64) { // also returned if requested and available. // Note: an error is only returned if retrieving the head header has failed. If there are no // retrievable blocks in the specified range then zero block count is returned with no error. -func (oracle *Oracle) resolveBlockRange(ctx context.Context, lastBlock rpc.BlockNumber, blocks int) (*types.Block, []*types.Receipt, uint64, int, error) { +func (oracle *Oracle) resolveBlockRange(ctx context.Context, reqEnd rpc.BlockNumber, blocks int) (*types.Block, []*types.Receipt, uint64, int, error) { var ( - headBlock rpc.BlockNumber + headBlock *types.Header pendingBlock *types.Block pendingReceipts types.Receipts + err error ) - // query either pending block or head header and set headBlock - if lastBlock == rpc.PendingBlockNumber { - if pendingBlock, pendingReceipts = oracle.backend.PendingBlockAndReceipts(); pendingBlock != nil { - lastBlock = rpc.BlockNumber(pendingBlock.NumberU64()) - headBlock = lastBlock - 1 - } else { - // pending block not supported by backend, process until latest block - lastBlock = rpc.LatestBlockNumber - blocks-- - if blocks == 0 { - return nil, nil, 0, 0, nil + + // Get the chain's current head. + if headBlock, err = oracle.backend.HeaderByNumber(ctx, rpc.LatestBlockNumber); err != nil { + return nil, nil, 0, 0, err + } + head := rpc.BlockNumber(headBlock.Number.Uint64()) + + // Fail if request block is beyond the chain's current head. + if head < reqEnd { + return nil, nil, 0, 0, fmt.Errorf("%w: requested %d, head %d", errRequestBeyondHead, reqEnd, head) + } + + // Resolve block tag. + if reqEnd < 0 { + var ( + resolved *types.Header + err error + ) + switch reqEnd { + case rpc.PendingBlockNumber: + if pendingBlock, pendingReceipts = oracle.backend.PendingBlockAndReceipts(); pendingBlock != nil { + resolved = pendingBlock.Header() + } else { + // Pending block not supported by backend, process only until latest block. + resolved = headBlock + + // Update total blocks to return to account for this. + blocks-- } + case rpc.LatestBlockNumber: + // Retrieved above. + resolved = headBlock + case rpc.SafeBlockNumber: + resolved, err = oracle.backend.HeaderByNumber(ctx, rpc.SafeBlockNumber) + case rpc.FinalizedBlockNumber: + resolved, err = oracle.backend.HeaderByNumber(ctx, rpc.FinalizedBlockNumber) + case rpc.EarliestBlockNumber: + resolved, err = oracle.backend.HeaderByNumber(ctx, rpc.EarliestBlockNumber) } - } - if pendingBlock == nil { - // if pending block is not fetched then we retrieve the head header to get the head block number - if latestHeader, err := oracle.backend.HeaderByNumber(ctx, rpc.LatestBlockNumber); err == nil { - headBlock = rpc.BlockNumber(latestHeader.Number.Uint64()) - } else { + if resolved == nil || err != nil { return nil, nil, 0, 0, err } + // Absolute number resolved. + reqEnd = rpc.BlockNumber(resolved.Number.Uint64()) } - if lastBlock == rpc.LatestBlockNumber { - lastBlock = headBlock - } else if pendingBlock == nil && lastBlock > headBlock { - return nil, nil, 0, 0, fmt.Errorf("%w: requested %d, head %d", errRequestBeyondHead, lastBlock, headBlock) + + // If there are no blocks to return, short circuit. + if blocks == 0 { + return nil, nil, 0, 0, nil } - // ensure not trying to retrieve before genesis - if rpc.BlockNumber(blocks) > lastBlock+1 { - blocks = int(lastBlock + 1) + // Ensure not trying to retrieve before genesis. + if int(reqEnd+1) < blocks { + blocks = int(reqEnd + 1) } - return pendingBlock, pendingReceipts, uint64(lastBlock), blocks, nil + return pendingBlock, pendingReceipts, uint64(reqEnd), blocks, nil } // FeeHistory returns data relevant for fee estimation based on the specified range of blocks. diff --git a/eth/gasprice/feehistory_test.go b/eth/gasprice/feehistory_test.go index c259eb0acf762..deece7f461500 100644 --- a/eth/gasprice/feehistory_test.go +++ b/eth/gasprice/feehistory_test.go @@ -50,6 +50,8 @@ func TestFeeHistory(t *testing.T) { {false, 1000, 1000, 2, rpc.PendingBlockNumber, nil, 32, 1, nil}, {true, 1000, 1000, 2, rpc.PendingBlockNumber, nil, 32, 2, nil}, {true, 1000, 1000, 2, rpc.PendingBlockNumber, []float64{0, 10}, 32, 2, nil}, + {false, 1000, 1000, 2, rpc.FinalizedBlockNumber, []float64{0, 10}, 24, 2, nil}, + {false, 1000, 1000, 2, rpc.SafeBlockNumber, []float64{0, 10}, 24, 2, nil}, } for i, c := range cases { config := Config{ diff --git a/eth/gasprice/gasprice_test.go b/eth/gasprice/gasprice_test.go index c0d3c6b6038e9..d405188f81945 100644 --- a/eth/gasprice/gasprice_test.go +++ b/eth/gasprice/gasprice_test.go @@ -45,6 +45,15 @@ func (b *testBackend) HeaderByNumber(ctx context.Context, number rpc.BlockNumber if number > testHead { return nil, nil } + if number == rpc.EarliestBlockNumber { + number = 0 + } + if number == rpc.FinalizedBlockNumber { + return b.chain.CurrentFinalizedBlock().Header(), nil + } + if number == rpc.SafeBlockNumber { + return b.chain.CurrentSafeBlock().Header(), nil + } if number == rpc.LatestBlockNumber { number = testHead } @@ -62,6 +71,15 @@ func (b *testBackend) BlockByNumber(ctx context.Context, number rpc.BlockNumber) if number > testHead { return nil, nil } + if number == rpc.EarliestBlockNumber { + number = 0 + } + if number == rpc.FinalizedBlockNumber { + return b.chain.CurrentFinalizedBlock(), nil + } + if number == rpc.SafeBlockNumber { + return b.chain.CurrentSafeBlock(), nil + } if number == rpc.LatestBlockNumber { number = testHead } @@ -108,12 +126,12 @@ func newTestBackend(t *testing.T, londonBlock *big.Int, pending bool) *testBacke ) config.LondonBlock = londonBlock config.ArrowGlacierBlock = londonBlock + config.GrayGlacierBlock = londonBlock + config.TerminalTotalDifficulty = common.Big0 engine := ethash.NewFaker() db := rawdb.NewMemoryDatabase() - genesis, err := gspec.Commit(db) - if err != nil { - t.Fatal(err) - } + genesis := gspec.MustCommit(db) + // Generate testing blocks blocks, _ := core.GenerateChain(gspec.Config, genesis, engine, db, testHead+1, func(i int, b *core.BlockGen) { b.SetCoinbase(common.Address{1}) @@ -143,12 +161,14 @@ func newTestBackend(t *testing.T, londonBlock *big.Int, pending bool) *testBacke }) // Construct testing chain diskdb := rawdb.NewMemoryDatabase() - gspec.Commit(diskdb) + gspec.MustCommit(diskdb) chain, err := core.NewBlockChain(diskdb, &core.CacheConfig{TrieCleanNoPrefetch: true}, gspec.Config, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to create local chain, %v", err) } chain.InsertChain(blocks) + chain.SetFinalized(chain.GetBlockByNumber(25)) + chain.SetSafe(chain.GetBlockByNumber(25)) return &testBackend{chain: chain, pending: pending} } diff --git a/eth/handler.go b/eth/handler.go index 54efe18d64a1b..143147b0c8153 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -191,11 +191,22 @@ func newHandler(config *handlerConfig) (*handler, error) { } } } - // Construct the downloader (long sync) and its backing state bloom if snap - // sync is requested. The downloader is responsible for deallocating the state - // bloom when it's done. + // Construct the downloader (long sync) h.downloader = downloader.New(h.checkpointNumber, config.Database, h.eventMux, h.chain, nil, h.removePeer, success) - + if ttd := h.chain.Config().TerminalTotalDifficulty; ttd != nil { + if h.chain.Config().TerminalTotalDifficultyPassed { + log.Info("Chain post-merge, sync via beacon client") + } else { + head := h.chain.CurrentBlock() + if td := h.chain.GetTd(head.Hash(), head.NumberU64()); td.Cmp(ttd) >= 0 { + log.Info("Chain post-TTD, sync via beacon client") + } else { + log.Warn("Chain pre-merge, sync via PoW (ensure beacon client is ready)") + } + } + } else if h.chain.Config().TerminalTotalDifficultyPassed { + log.Error("Chain configured post-merge, but without TTD. Are you debugging sync?") + } // Construct the fetcher (short sync) validator := func(header *types.Header) error { // All the block fetcher activities should be disabled @@ -249,7 +260,7 @@ func newHandler(config *handlerConfig) (*handler, error) { // out a way yet where nodes can decide unilaterally whether the network is new // or not. This should be fixed if we figure out a solution. if atomic.LoadUint32(&h.snapSync) == 1 { - log.Warn("Fast syncing, discarded propagated block", "number", blocks[0].Number(), "hash", blocks[0].Hash()) + log.Warn("Snap syncing, discarded propagated block", "number", blocks[0].Number(), "hash", blocks[0].Hash()) return 0, nil } if h.merger.TDDReached() { @@ -296,7 +307,7 @@ func newHandler(config *handlerConfig) (*handler, error) { } // runEthPeer registers an eth peer into the joint eth/snap peerset, adds it to -// various subsistems and starts handling messages. +// various subsystems and starts handling messages. func (h *handler) runEthPeer(peer *eth.Peer, handler eth.Handler) error { // If the peer has a `snap` extension, wait for it to connect so we can have // a uniform initialization/teardown mechanism @@ -380,11 +391,16 @@ func (h *handler) runEthPeer(peer *eth.Peer, handler eth.Handler) error { if h.checkpointHash != (common.Hash{}) { // Request the peer's checkpoint header for chain height/weight validation resCh := make(chan *eth.Response) - if _, err := peer.RequestHeadersByNumber(h.checkpointNumber, 1, 0, false, resCh); err != nil { + + req, err := peer.RequestHeadersByNumber(h.checkpointNumber, 1, 0, false, resCh) + if err != nil { return err } // Start a timer to disconnect if the peer doesn't reply in time go func() { + // Ensure the request gets cancelled in case of error/drop + defer req.Close() + timeout := time.NewTimer(syncChallengeTimeout) defer timeout.Stop() @@ -426,10 +442,15 @@ func (h *handler) runEthPeer(peer *eth.Peer, handler eth.Handler) error { // If we have any explicit peer required block hashes, request them for number, hash := range h.requiredBlocks { resCh := make(chan *eth.Response) - if _, err := peer.RequestHeadersByNumber(number, 1, 0, false, resCh); err != nil { + + req, err := peer.RequestHeadersByNumber(number, 1, 0, false, resCh) + if err != nil { return err } - go func(number uint64, hash common.Hash) { + go func(number uint64, hash common.Hash, req *eth.Request) { + // Ensure the request gets cancelled in case of error/drop + defer req.Close() + timeout := time.NewTimer(syncChallengeTimeout) defer timeout.Stop() @@ -458,7 +479,7 @@ func (h *handler) runEthPeer(peer *eth.Peer, handler eth.Handler) error { peer.Log().Warn("Required block challenge timed out, dropping", "addr", peer.RemoteAddr(), "type", peer.Name()) h.removePeer(peer.ID()) } - }(number, hash) + }(number, hash, req) } // Handle incoming messages until the connection is torn down return handler(peer) diff --git a/eth/handler_eth_test.go b/eth/handler_eth_test.go index dffbfbe612a2b..453bc5e98e318 100644 --- a/eth/handler_eth_test.go +++ b/eth/handler_eth_test.go @@ -490,7 +490,6 @@ func TestCheckpointChallenge(t *testing.T) { } func testCheckpointChallenge(t *testing.T, syncmode downloader.SyncMode, checkpoint bool, timeout bool, empty bool, match bool, drop bool) { - // Reduce the checkpoint handshake challenge timeout defer func(old time.Duration) { syncChallengeTimeout = old }(syncChallengeTimeout) syncChallengeTimeout = 250 * time.Millisecond diff --git a/eth/peer.go b/eth/peer.go index 024a6e619371f..55e5f0046206a 100644 --- a/eth/peer.go +++ b/eth/peer.go @@ -34,8 +34,7 @@ type ethPeerInfo struct { // ethPeer is a wrapper around eth.Peer to maintain a few extra metadata. type ethPeer struct { *eth.Peer - snapExt *snapPeer // Satellite `snap` connection - snapWait chan struct{} // Notification channel for snap connections + snapExt *snapPeer // Satellite `snap` connection } // info gathers and returns some `eth` protocol metadata known about a peer. diff --git a/eth/peerset.go b/eth/peerset.go index 3e54a481e36b4..b9cc1e03aca3d 100644 --- a/eth/peerset.go +++ b/eth/peerset.go @@ -41,7 +41,7 @@ var ( errPeerNotRegistered = errors.New("peer not registered") // errSnapWithoutEth is returned if a peer attempts to connect only on the - // snap protocol without advertizing the eth main protocol. + // snap protocol without advertising the eth main protocol. errSnapWithoutEth = errors.New("peer connected on snap without compatible eth support") ) diff --git a/eth/protocols/eth/broadcast.go b/eth/protocols/eth/broadcast.go index 09330cfdf320a..6fc15f136bff9 100644 --- a/eth/protocols/eth/broadcast.go +++ b/eth/protocols/eth/broadcast.go @@ -36,7 +36,7 @@ type blockPropagation struct { td *big.Int } -// broadcastBlocks is a write loop that multiplexes blocks and block accouncements +// broadcastBlocks is a write loop that multiplexes blocks and block announcements // to the remote peer. The goal is to have an async writer that does not lock up // node internals and at the same time rate limits queued data. func (p *Peer) broadcastBlocks() { diff --git a/eth/protocols/eth/dispatcher.go b/eth/protocols/eth/dispatcher.go index bf88d400d4a0c..65a935d555482 100644 --- a/eth/protocols/eth/dispatcher.go +++ b/eth/protocols/eth/dispatcher.go @@ -224,7 +224,7 @@ func (p *Peer) dispatcher() { switch { case res.Req == nil: // Response arrived with an untracked ID. Since even cancelled - // requests are tracked until fulfilment, a dangling repsponse + // requests are tracked until fulfilment, a dangling response // means the remote peer implements the protocol badly. resOp.fail <- errDanglingResponse diff --git a/eth/protocols/eth/handler.go b/eth/protocols/eth/handler.go index 81d45d8b8fcf0..3a0b21c30bdb2 100644 --- a/eth/protocols/eth/handler.go +++ b/eth/protocols/eth/handler.go @@ -181,6 +181,21 @@ var eth66 = map[uint64]msgHandler{ PooledTransactionsMsg: handlePooledTransactions66, } +var eth67 = map[uint64]msgHandler{ + NewBlockHashesMsg: handleNewBlockhashes, + NewBlockMsg: handleNewBlock, + TransactionsMsg: handleTransactions, + NewPooledTransactionHashesMsg: handleNewPooledTransactionHashes, + GetBlockHeadersMsg: handleGetBlockHeaders66, + BlockHeadersMsg: handleBlockHeaders66, + GetBlockBodiesMsg: handleGetBlockBodies66, + BlockBodiesMsg: handleBlockBodies66, + GetReceiptsMsg: handleGetReceipts66, + ReceiptsMsg: handleReceipts66, + GetPooledTransactionsMsg: handleGetPooledTransactions66, + PooledTransactionsMsg: handlePooledTransactions66, +} + // handleMessage is invoked whenever an inbound message is received from a remote // peer. The remote connection is torn down upon returning any error. func handleMessage(backend Backend, peer *Peer) error { @@ -195,9 +210,9 @@ func handleMessage(backend Backend, peer *Peer) error { defer msg.Discard() var handlers = eth66 - //if peer.Version() >= ETH67 { // Left in as a sample when new protocol is added - // handlers = eth67 - //} + if peer.Version() >= ETH67 { + handlers = eth67 + } // Track the amount of time it takes to serve the request and run the handler if metrics.Enabled { diff --git a/eth/protocols/eth/handler_test.go b/eth/protocols/eth/handler_test.go index bf836e8f51323..2707a420bc6a1 100644 --- a/eth/protocols/eth/handler_test.go +++ b/eth/protocols/eth/handler_test.go @@ -94,7 +94,7 @@ func (b *testBackend) Chain() *core.BlockChain { return b.chain } func (b *testBackend) TxPool() TxPool { return b.txpool } func (b *testBackend) RunPeer(peer *Peer, handler Handler) error { - // Normally the backend would do peer mainentance and handshakes. All that + // Normally the backend would do peer maintenance and handshakes. All that // is omitted and we will just give control back to the handler. return handler(peer) } diff --git a/eth/protocols/eth/peer.go b/eth/protocols/eth/peer.go index 22674d65b041f..a23726384d70a 100644 --- a/eth/protocols/eth/peer.go +++ b/eth/protocols/eth/peer.go @@ -133,7 +133,7 @@ func (p *Peer) ID() string { return p.id } -// Version retrieves the peer's negoatiated `eth` protocol version. +// Version retrieves the peer's negotiated `eth` protocol version. func (p *Peer) Version() uint { return p.version } diff --git a/eth/protocols/eth/protocol.go b/eth/protocols/eth/protocol.go index 24b65f01dd96a..f6fac42780805 100644 --- a/eth/protocols/eth/protocol.go +++ b/eth/protocols/eth/protocol.go @@ -31,6 +31,7 @@ import ( // Constants to match up protocol versions and messages const ( ETH66 = 66 + ETH67 = 67 ) // ProtocolName is the official short name of the `eth` protocol used during @@ -39,11 +40,11 @@ const ProtocolName = "eth" // ProtocolVersions are the supported versions of the `eth` protocol (first // is primary). -var ProtocolVersions = []uint{ETH66} +var ProtocolVersions = []uint{ETH67, ETH66} // protocolLengths are the number of implemented message corresponding to // different protocol versions. -var protocolLengths = map[uint]uint64{ETH66: 17} +var protocolLengths = map[uint]uint64{ETH67: 17, ETH66: 17} // maxMessageSize is the maximum cap on the size of a protocol message. const maxMessageSize = 10 * 1024 * 1024 diff --git a/eth/protocols/eth/protocol_test.go b/eth/protocols/eth/protocol_test.go index 5ca8957741210..a86fbb0a6906a 100644 --- a/eth/protocols/eth/protocol_test.go +++ b/eth/protocols/eth/protocol_test.go @@ -115,12 +115,10 @@ func TestEth66EmptyMessages(t *testing.T) { t.Errorf("test %d, type %T, have\n\t%x\nwant\n\t%x", i, msg, have, want) } } - } // TestEth66Messages tests the encoding of all redefined eth66 messages func TestEth66Messages(t *testing.T) { - // Some basic structs used during testing var ( header *types.Header diff --git a/eth/protocols/snap/handler.go b/eth/protocols/snap/handler.go index 23638ef888849..41380d96f5717 100644 --- a/eth/protocols/snap/handler.go +++ b/eth/protocols/snap/handler.go @@ -23,14 +23,12 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/light" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/p2p/enr" - "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" ) @@ -285,7 +283,7 @@ func ServiceGetAccountRangeQuery(chain *core.BlockChain, req *GetAccountRangePac req.Bytes = softResponseLimit } // Retrieve the requested state and bail out if non existent - tr, err := trie.New(req.Root, chain.StateCache().TrieDB()) + tr, err := trie.New(common.Hash{}, req.Root, chain.StateCache().TrieDB()) if err != nil { return nil, nil } @@ -415,15 +413,15 @@ func ServiceGetStorageRangesQuery(chain *core.BlockChain, req *GetStorageRangesP if origin != (common.Hash{}) || (abort && len(storage) > 0) { // Request started at a non-zero hash or was capped prematurely, add // the endpoint Merkle proofs - accTrie, err := trie.New(req.Root, chain.StateCache().TrieDB()) + accTrie, err := trie.NewStateTrie(common.Hash{}, req.Root, chain.StateCache().TrieDB()) if err != nil { return nil, nil } - var acc types.StateAccount - if err := rlp.DecodeBytes(accTrie.Get(account[:]), &acc); err != nil { + acc, err := accTrie.TryGetAccountWithPreHashedKey(account[:]) + if err != nil || acc == nil { return nil, nil } - stTrie, err := trie.New(acc.Root, chain.StateCache().TrieDB()) + stTrie, err := trie.NewStateTrie(account, acc.Root, chain.StateCache().TrieDB()) if err != nil { return nil, nil } @@ -489,7 +487,7 @@ func ServiceGetTrieNodesQuery(chain *core.BlockChain, req *GetTrieNodesPacket, s // Make sure we have the state associated with the request triedb := chain.StateCache().TrieDB() - accTrie, err := trie.NewSecure(req.Root, triedb) + accTrie, err := trie.NewStateTrie(common.Hash{}, req.Root, triedb) if err != nil { // We don't have the requested state available, bail out return nil, nil @@ -506,7 +504,7 @@ func ServiceGetTrieNodesQuery(chain *core.BlockChain, req *GetTrieNodesPacket, s var ( nodes [][]byte bytes uint64 - loads int // Trie hash expansions to cound database reads + loads int // Trie hash expansions to count database reads ) for _, pathset := range req.Paths { switch len(pathset) { @@ -531,7 +529,7 @@ func ServiceGetTrieNodesQuery(chain *core.BlockChain, req *GetTrieNodesPacket, s if err != nil || account == nil { break } - stTrie, err := trie.NewSecure(common.BytesToHash(account.Root), triedb) + stTrie, err := trie.NewStateTrie(common.BytesToHash(pathset[0]), common.BytesToHash(account.Root), triedb) loads++ // always account database reads, even for failures if err != nil { break diff --git a/eth/protocols/snap/peer.go b/eth/protocols/snap/peer.go index 87a62d2f8a413..235d499ffdc98 100644 --- a/eth/protocols/snap/peer.go +++ b/eth/protocols/snap/peer.go @@ -61,12 +61,12 @@ func (p *Peer) ID() string { return p.id } -// Version retrieves the peer's negoatiated `snap` protocol version. +// Version retrieves the peer's negotiated `snap` protocol version. func (p *Peer) Version() uint { return p.version } -// Log overrides the P2P logget with the higher level one containing only the id. +// Log overrides the P2P logger with the higher level one containing only the id. func (p *Peer) Log() log.Logger { return p.logger } @@ -87,7 +87,7 @@ func (p *Peer) RequestAccountRange(id uint64, root common.Hash, origin, limit co } // RequestStorageRange fetches a batch of storage slots belonging to one or more -// accounts. If slots from only one accout is requested, an origin marker may also +// accounts. If slots from only one account is requested, an origin marker may also // be used to retrieve from there. func (p *Peer) RequestStorageRanges(id uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, bytes uint64) error { if len(accounts) == 1 && origin != nil { @@ -119,7 +119,7 @@ func (p *Peer) RequestByteCodes(id uint64, hashes []common.Hash, bytes uint64) e } // RequestTrieNodes fetches a batch of account or storage trie nodes rooted in -// a specificstate trie. +// a specific state trie. func (p *Peer) RequestTrieNodes(id uint64, root common.Hash, paths []TrieNodePathSet, bytes uint64) error { p.logger.Trace("Fetching set of trie nodes", "reqid", id, "root", root, "pathsets", len(paths), "bytes", common.StorageSize(bytes)) diff --git a/eth/protocols/snap/sort_test.go b/eth/protocols/snap/sort_test.go index c625be09ea549..be0a8c5706968 100644 --- a/eth/protocols/snap/sort_test.go +++ b/eth/protocols/snap/sort_test.go @@ -22,7 +22,6 @@ import ( "testing" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/trie" ) func hexToNibbles(s string) []byte { @@ -38,22 +37,17 @@ func hexToNibbles(s string) []byte { } func TestRequestSorting(t *testing.T) { - // - Path 0x9 -> {0x19} // - Path 0x99 -> {0x0099} // - Path 0x01234567890123456789012345678901012345678901234567890123456789019 -> {0x0123456789012345678901234567890101234567890123456789012345678901, 0x19} // - Path 0x012345678901234567890123456789010123456789012345678901234567890199 -> {0x0123456789012345678901234567890101234567890123456789012345678901, 0x0099} - var f = func(path string) (trie.SyncPath, TrieNodePathSet, common.Hash) { + var f = func(path string) string { data := hexToNibbles(path) - sp := trie.NewSyncPath(data) - tnps := TrieNodePathSet([][]byte(sp)) - hash := common.Hash{} - return sp, tnps, hash + return string(data) } var ( - hashes []common.Hash - paths []trie.SyncPath - pathsets []TrieNodePathSet + hashes []common.Hash + paths []string ) for _, x := range []string{ "0x9", @@ -67,16 +61,14 @@ func TestRequestSorting(t *testing.T) { "0x01234567890123456789012345678901012345678901234567890123456789010", "0x01234567890123456789012345678901012345678901234567890123456789011", } { - sp, tnps, hash := f(x) - hashes = append(hashes, hash) - paths = append(paths, sp) - pathsets = append(pathsets, tnps) + paths = append(paths, f(x)) + hashes = append(hashes, common.Hash{}) } - _, paths, pathsets = sortByAccountPath(hashes, paths) + _, _, syncPaths, pathsets := sortByAccountPath(paths, hashes) { var b = new(bytes.Buffer) - for i := 0; i < len(paths); i++ { - fmt.Fprintf(b, "\n%d. paths %x", i, paths[i]) + for i := 0; i < len(syncPaths); i++ { + fmt.Fprintf(b, "\n%d. paths %x", i, syncPaths[i]) } want := ` 0. paths [0099] diff --git a/eth/protocols/snap/sync.go b/eth/protocols/snap/sync.go index 415253c839be2..eb8260bf7c97a 100644 --- a/eth/protocols/snap/sync.go +++ b/eth/protocols/snap/sync.go @@ -21,10 +21,12 @@ import ( "encoding/json" "errors" "fmt" + gomath "math" "math/big" "math/rand" "sort" "sync" + "sync/atomic" "time" "github.com/ethereum/go-ethereum/common" @@ -78,6 +80,29 @@ const ( // and waste round trip times. If it's too high, we're capping responses and // waste bandwidth. maxTrieRequestCount = maxRequestSize / 512 + + // trienodeHealRateMeasurementImpact is the impact a single measurement has on + // the local node's trienode processing capacity. A value closer to 0 reacts + // slower to sudden changes, but it is also more stable against temporary hiccups. + trienodeHealRateMeasurementImpact = 0.005 + + // minTrienodeHealThrottle is the minimum divisor for throttling trie node + // heal requests to avoid overloading the local node and exessively expanding + // the state trie bedth wise. + minTrienodeHealThrottle = 1 + + // maxTrienodeHealThrottle is the maximum divisor for throttling trie node + // heal requests to avoid overloading the local node and exessively expanding + // the state trie bedth wise. + maxTrienodeHealThrottle = maxTrieRequestCount + + // trienodeHealThrottleIncrease is the multiplier for the throttle when the + // rate of arriving data is higher than the rate of processing it. + trienodeHealThrottleIncrease = 1.33 + + // trienodeHealThrottleDecrease is the divisor for the throttle when the + // rate of arriving data is lower than the rate of processing it. + trienodeHealThrottleDecrease = 1.25 ) var ( @@ -230,8 +255,8 @@ type trienodeHealRequest struct { timeout *time.Timer // Timer to track delivery timeout stale chan struct{} // Channel to signal the request was dropped - hashes []common.Hash // Trie node hashes to validate responses - paths []trie.SyncPath // Trie node paths requested for rescheduling + paths []string // Trie node paths for identifying trie node + hashes []common.Hash // Trie node hashes to validate responses task *healTask // Task which this request is filling (only access fields through the runloop!!) } @@ -240,9 +265,9 @@ type trienodeHealRequest struct { type trienodeHealResponse struct { task *healTask // Task which this request is filling - hashes []common.Hash // Hashes of the trie nodes to avoid double hashing - paths []trie.SyncPath // Trie node paths requested for rescheduling missing ones - nodes [][]byte // Actual trie nodes to store into the database (nil = missing) + paths []string // Paths of the trie nodes + hashes []common.Hash // Hashes of the trie nodes to avoid double hashing + nodes [][]byte // Actual trie nodes to store into the database (nil = missing) } // bytecodeHealRequest tracks a pending bytecode request to ensure responses are to @@ -321,8 +346,8 @@ type storageTask struct { type healTask struct { scheduler *trie.Sync // State trie sync scheduler defining the tasks - trieTasks map[common.Hash]trie.SyncPath // Set of trie node tasks currently queued for retrieval - codeTasks map[common.Hash]struct{} // Set of byte code tasks currently queued for retrieval + trieTasks map[string]common.Hash // Set of trie node tasks currently queued for retrieval, indexed by node path + codeTasks map[common.Hash]struct{} // Set of byte code tasks currently queued for retrieval, indexed by code hash } // SyncProgress is a database entry to allow suspending and resuming a snapshot state @@ -365,7 +390,7 @@ type SyncPeer interface { RequestAccountRange(id uint64, root, origin, limit common.Hash, bytes uint64) error // RequestStorageRanges fetches a batch of storage slots belonging to one or - // more accounts. If slots from only one accout is requested, an origin marker + // more accounts. If slots from only one account is requested, an origin marker // may also be used to retrieve from there. RequestStorageRanges(id uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, bytes uint64) error @@ -373,7 +398,7 @@ type SyncPeer interface { RequestByteCodes(id uint64, hashes []common.Hash, bytes uint64) error // RequestTrieNodes fetches a batch of account or storage trie nodes rooted in - // a specificstate trie. + // a specific state trie. RequestTrieNodes(id uint64, root common.Hash, paths []TrieNodePathSet, bytes uint64) error // Log retrieves the peer's own contextual logger. @@ -431,6 +456,11 @@ type Syncer struct { trienodeHealReqs map[uint64]*trienodeHealRequest // Trie node requests currently running bytecodeHealReqs map[uint64]*bytecodeHealRequest // Bytecode requests currently running + trienodeHealRate float64 // Average heal rate for processing trie node data + trienodeHealPend uint64 // Number of trie nodes currently pending for processing + trienodeHealThrottle float64 // Divisor for throttling the amount of trienode heal data requested + trienodeHealThrottled time.Time // Timestamp the last time the throttle was updated + trienodeHealSynced uint64 // Number of state trie nodes downloaded trienodeHealBytes common.StorageSize // Number of state trie bytes persisted to disk trienodeHealDups uint64 // Number of state trie nodes already processed @@ -476,9 +506,10 @@ func NewSyncer(db ethdb.KeyValueStore) *Syncer { trienodeHealIdlers: make(map[string]struct{}), bytecodeHealIdlers: make(map[string]struct{}), - trienodeHealReqs: make(map[uint64]*trienodeHealRequest), - bytecodeHealReqs: make(map[uint64]*bytecodeHealRequest), - stateWriter: db.NewBatch(), + trienodeHealReqs: make(map[uint64]*trienodeHealRequest), + bytecodeHealReqs: make(map[uint64]*bytecodeHealRequest), + trienodeHealThrottle: maxTrienodeHealThrottle, // Tune downward instead of insta-filling with junk + stateWriter: db.NewBatch(), extProgress: new(SyncProgress), } @@ -540,7 +571,7 @@ func (s *Syncer) Unregister(id string) error { return nil } -// Sync starts (or resumes a previous) sync cycle to iterate over an state trie +// Sync starts (or resumes a previous) sync cycle to iterate over a state trie // with the given root and reconstruct the nodes based on the snapshot leaves. // Previously downloaded segments will not be redownloaded of fixed, rather any // errors will be healed after the leaves are fully accumulated. @@ -551,7 +582,7 @@ func (s *Syncer) Sync(root common.Hash, cancel chan struct{}) error { s.root = root s.healer = &healTask{ scheduler: state.NewStateSync(root, s.db, s.onHealState), - trieTasks: make(map[common.Hash]trie.SyncPath), + trieTasks: make(map[string]common.Hash), codeTasks: make(map[common.Hash]struct{}), } s.statelessPeers = make(map[string]struct{}) @@ -712,7 +743,7 @@ func (s *Syncer) loadSyncStatus() { } task.genTrie = trie.NewStackTrie(task.genBatch) - for _, subtasks := range task.SubTasks { + for accountHash, subtasks := range task.SubTasks { for _, subtask := range subtasks { subtask.genBatch = ethdb.HookedBatch{ Batch: s.db.NewBatch(), @@ -720,7 +751,7 @@ func (s *Syncer) loadSyncStatus() { s.storageBytes += common.StorageSize(len(key) + len(value)) }, } - subtask.genTrie = trie.NewStackTrie(subtask.genBatch) + subtask.genTrie = trie.NewStackTrieWithOwner(subtask.genBatch, accountHash) } } } @@ -743,7 +774,7 @@ func (s *Syncer) loadSyncStatus() { return } } - // Either we've failed to decode the previus state, or there was none. + // Either we've failed to decode the previous state, or there was none. // Start a fresh sync by chunking up the account range and scheduling // them for retrieval. s.tasks = nil @@ -1183,10 +1214,10 @@ func (s *Syncer) assignStorageTasks(success chan *storageResponse, fail chan *st } if subtask == nil { // No large contract required retrieval, but small ones available - for acccount, root := range task.stateTasks { - delete(task.stateTasks, acccount) + for account, root := range task.stateTasks { + delete(task.stateTasks, account) - accounts = append(accounts, acccount) + accounts = append(accounts, account) roots = append(roots, root) if len(accounts) >= storageSets { @@ -1280,9 +1311,9 @@ func (s *Syncer) assignTrienodeHealTasks(success chan *trienodeHealResponse, fai want = maxTrieRequestCount + maxCodeRequestCount ) if have < want { - nodes, paths, codes := s.healer.scheduler.Missing(want - have) - for i, hash := range nodes { - s.healer.trieTasks[hash] = paths[i] + paths, hashes, codes := s.healer.scheduler.Missing(want - have) + for i, path := range paths { + s.healer.trieTasks[path] = hashes[i] } for _, hash := range codes { s.healer.codeTasks[hash] = struct{}{} @@ -1321,23 +1352,26 @@ func (s *Syncer) assignTrienodeHealTasks(success chan *trienodeHealResponse, fai if cap > maxTrieRequestCount { cap = maxTrieRequestCount } + cap = int(float64(cap) / s.trienodeHealThrottle) + if cap <= 0 { + cap = 1 + } var ( hashes = make([]common.Hash, 0, cap) - paths = make([]trie.SyncPath, 0, cap) + paths = make([]string, 0, cap) pathsets = make([]TrieNodePathSet, 0, cap) ) - for hash, pathset := range s.healer.trieTasks { - delete(s.healer.trieTasks, hash) + for path, hash := range s.healer.trieTasks { + delete(s.healer.trieTasks, path) + paths = append(paths, path) hashes = append(hashes, hash) - paths = append(paths, pathset) - - if len(hashes) >= cap { + if len(paths) >= cap { break } } // Group requests by account hash - hashes, paths, pathsets = sortByAccountPath(hashes, paths) + paths, hashes, _, pathsets = sortByAccountPath(paths, hashes) req := &trienodeHealRequest{ peer: idle, id: reqid, @@ -1346,8 +1380,8 @@ func (s *Syncer) assignTrienodeHealTasks(success chan *trienodeHealResponse, fai revert: fail, cancel: cancel, stale: make(chan struct{}), - hashes: hashes, paths: paths, + hashes: hashes, task: s.healer, } req.timeout = time.AfterFunc(s.rates.TargetTimeout(), func() { @@ -1405,9 +1439,9 @@ func (s *Syncer) assignBytecodeHealTasks(success chan *bytecodeHealResponse, fai want = maxTrieRequestCount + maxCodeRequestCount ) if have < want { - nodes, paths, codes := s.healer.scheduler.Missing(want - have) - for i, hash := range nodes { - s.healer.trieTasks[hash] = paths[i] + paths, hashes, codes := s.healer.scheduler.Missing(want - have) + for i, path := range paths { + s.healer.trieTasks[path] = hashes[i] } for _, hash := range codes { s.healer.codeTasks[hash] = struct{}{} @@ -1487,7 +1521,7 @@ func (s *Syncer) assignBytecodeHealTasks(success chan *bytecodeHealResponse, fai } } -// revertRequests locates all the currently pending reuqests from a particular +// revertRequests locates all the currently pending requests from a particular // peer and reverts them, rescheduling for others to fulfill. func (s *Syncer) revertRequests(peer string) { // Gather the requests first, revertals need the lock too @@ -1576,7 +1610,7 @@ func (s *Syncer) revertAccountRequest(req *accountRequest) { s.lock.Unlock() // If there's a timeout timer still running, abort it and mark the account - // task as not-pending, ready for resheduling + // task as not-pending, ready for rescheduling req.timeout.Stop() if req.task.req == req { req.task.req = nil @@ -1617,7 +1651,7 @@ func (s *Syncer) revertBytecodeRequest(req *bytecodeRequest) { s.lock.Unlock() // If there's a timeout timer still running, abort it and mark the code - // retrievals as not-pending, ready for resheduling + // retrievals as not-pending, ready for rescheduling req.timeout.Stop() for _, hash := range req.hashes { req.task.codeTasks[hash] = struct{}{} @@ -1658,7 +1692,7 @@ func (s *Syncer) revertStorageRequest(req *storageRequest) { s.lock.Unlock() // If there's a timeout timer still running, abort it and mark the storage - // task as not-pending, ready for resheduling + // task as not-pending, ready for rescheduling req.timeout.Stop() if req.subTask != nil { req.subTask.req = nil @@ -1703,10 +1737,10 @@ func (s *Syncer) revertTrienodeHealRequest(req *trienodeHealRequest) { s.lock.Unlock() // If there's a timeout timer still running, abort it and mark the trie node - // retrievals as not-pending, ready for resheduling + // retrievals as not-pending, ready for rescheduling req.timeout.Stop() - for i, hash := range req.hashes { - req.task.trieTasks[hash] = req.paths[i] + for i, path := range req.paths { + req.task.trieTasks[path] = req.hashes[i] } } @@ -1744,7 +1778,7 @@ func (s *Syncer) revertBytecodeHealRequest(req *bytecodeHealRequest) { s.lock.Unlock() // If there's a timeout timer still running, abort it and mark the code - // retrievals as not-pending, ready for resheduling + // retrievals as not-pending, ready for rescheduling req.timeout.Stop() for _, hash := range req.hashes { req.task.codeTasks[hash] = struct{}{} @@ -1971,7 +2005,7 @@ func (s *Syncer) processStorageResponse(res *storageResponse) { Last: r.End(), root: acc.Root, genBatch: batch, - genTrie: trie.NewStackTrie(batch), + genTrie: trie.NewStackTrieWithOwner(batch, account), }) for r.Next() { batch := ethdb.HookedBatch{ @@ -1985,7 +2019,7 @@ func (s *Syncer) processStorageResponse(res *storageResponse) { Last: r.End(), root: acc.Root, genBatch: batch, - genTrie: trie.NewStackTrie(batch), + genTrie: trie.NewStackTrieWithOwner(batch, account), }) } for _, task := range tasks { @@ -2030,13 +2064,13 @@ func (s *Syncer) processStorageResponse(res *storageResponse) { slots += len(res.hashes[i]) if i < len(res.hashes)-1 || res.subTask == nil { - tr := trie.NewStackTrie(batch) + tr := trie.NewStackTrieWithOwner(batch, account) for j := 0; j < len(res.hashes[i]); j++ { tr.Update(res.hashes[i][j][:], res.slots[i][j]) } tr.Commit() } - // Persist the received storage segements. These flat state maybe + // Persist the received storage segments. These flat state maybe // outdated during the sync, but it can be fixed later during the // snapshot generation. for j := 0; j < len(res.hashes[i]); j++ { @@ -2091,19 +2125,25 @@ func (s *Syncer) processStorageResponse(res *storageResponse) { // processTrienodeHealResponse integrates an already validated trienode response // into the healer tasks. func (s *Syncer) processTrienodeHealResponse(res *trienodeHealResponse) { + var ( + start = time.Now() + fills int + ) for i, hash := range res.hashes { node := res.nodes[i] // If the trie node was not delivered, reschedule it if node == nil { - res.task.trieTasks[hash] = res.paths[i] + res.task.trieTasks[res.paths[i]] = res.hashes[i] continue } + fills++ + // Push the trie node into the state syncer s.trienodeHealSynced++ s.trienodeHealBytes += common.StorageSize(len(node)) - err := s.healer.scheduler.Process(trie.SyncResult{Hash: hash, Data: node}) + err := s.healer.scheduler.ProcessNode(trie.NodeSyncResult{Path: res.paths[i], Data: node}) switch err { case nil: case trie.ErrAlreadyProcessed: @@ -2122,6 +2162,50 @@ func (s *Syncer) processTrienodeHealResponse(res *trienodeHealResponse) { log.Crit("Failed to persist healing data", "err", err) } log.Debug("Persisted set of healing data", "type", "trienodes", "bytes", common.StorageSize(batch.ValueSize())) + + // Calculate the processing rate of one filled trie node + rate := float64(fills) / (float64(time.Since(start)) / float64(time.Second)) + + // Update the currently measured trienode queueing and processing throughput. + // + // The processing rate needs to be updated uniformly independent if we've + // processed 1x100 trie nodes or 100x1 to keep the rate consistent even in + // the face of varying network packets. As such, we cannot just measure the + // time it took to process N trie nodes and update once, we need one update + // per trie node. + // + // Naively, that would be: + // + // for i:=0; i time.Second { + // Periodically adjust the trie node throttler + if float64(pending) > 2*s.trienodeHealRate { + s.trienodeHealThrottle *= trienodeHealThrottleIncrease + } else { + s.trienodeHealThrottle /= trienodeHealThrottleDecrease + } + if s.trienodeHealThrottle > maxTrienodeHealThrottle { + s.trienodeHealThrottle = maxTrienodeHealThrottle + } else if s.trienodeHealThrottle < minTrienodeHealThrottle { + s.trienodeHealThrottle = minTrienodeHealThrottle + } + s.trienodeHealThrottled = time.Now() + + log.Debug("Updated trie node heal throttler", "rate", s.trienodeHealRate, "pending", pending, "throttle", s.trienodeHealThrottle) + } } // processBytecodeHealResponse integrates an already validated bytecode response @@ -2139,7 +2223,7 @@ func (s *Syncer) processBytecodeHealResponse(res *bytecodeHealResponse) { s.bytecodeHealSynced++ s.bytecodeHealBytes += common.StorageSize(len(node)) - err := s.healer.scheduler.Process(trie.SyncResult{Hash: hash, Data: node}) + err := s.healer.scheduler.ProcessCode(trie.CodeSyncResult{Hash: hash, Data: node}) switch err { case nil: case trie.ErrAlreadyProcessed: @@ -2171,7 +2255,7 @@ func (s *Syncer) forwardAccountTask(task *accountTask) { } task.res = nil - // Persist the received account segements. These flat state maybe + // Persist the received account segments. These flat state maybe // outdated during the sync, but it can be fixed later during the // snapshot generation. oldAccountBytes := s.accountBytes @@ -2249,14 +2333,18 @@ func (s *Syncer) OnAccounts(peer SyncPeer, id uint64, hashes []common.Hash, acco // Whether or not the response is valid, we can mark the peer as idle and // notify the scheduler to assign a new task. If the response is invalid, // we'll drop the peer in a bit. + defer func() { + s.lock.Lock() + defer s.lock.Unlock() + if _, ok := s.peers[peer.ID()]; ok { + s.accountIdlers[peer.ID()] = struct{}{} + } + select { + case s.update <- struct{}{}: + default: + } + }() s.lock.Lock() - if _, ok := s.peers[peer.ID()]; ok { - s.accountIdlers[peer.ID()] = struct{}{} - } - select { - case s.update <- struct{}{}: - default: - } // Ensure the response is for a valid request req, ok := s.accountReqs[id] if !ok { @@ -2361,14 +2449,18 @@ func (s *Syncer) onByteCodes(peer SyncPeer, id uint64, bytecodes [][]byte) error // Whether or not the response is valid, we can mark the peer as idle and // notify the scheduler to assign a new task. If the response is invalid, // we'll drop the peer in a bit. + defer func() { + s.lock.Lock() + defer s.lock.Unlock() + if _, ok := s.peers[peer.ID()]; ok { + s.bytecodeIdlers[peer.ID()] = struct{}{} + } + select { + case s.update <- struct{}{}: + default: + } + }() s.lock.Lock() - if _, ok := s.peers[peer.ID()]; ok { - s.bytecodeIdlers[peer.ID()] = struct{}{} - } - select { - case s.update <- struct{}{}: - default: - } // Ensure the response is for a valid request req, ok := s.bytecodeReqs[id] if !ok { @@ -2470,14 +2562,18 @@ func (s *Syncer) OnStorage(peer SyncPeer, id uint64, hashes [][]common.Hash, slo // Whether or not the response is valid, we can mark the peer as idle and // notify the scheduler to assign a new task. If the response is invalid, // we'll drop the peer in a bit. + defer func() { + s.lock.Lock() + defer s.lock.Unlock() + if _, ok := s.peers[peer.ID()]; ok { + s.storageIdlers[peer.ID()] = struct{}{} + } + select { + case s.update <- struct{}{}: + default: + } + }() s.lock.Lock() - if _, ok := s.peers[peer.ID()]; ok { - s.storageIdlers[peer.ID()] = struct{}{} - } - select { - case s.update <- struct{}{}: - default: - } // Ensure the response is for a valid request req, ok := s.storageReqs[id] if !ok { @@ -2597,14 +2693,18 @@ func (s *Syncer) OnTrieNodes(peer SyncPeer, id uint64, trienodes [][]byte) error // Whether or not the response is valid, we can mark the peer as idle and // notify the scheduler to assign a new task. If the response is invalid, // we'll drop the peer in a bit. + defer func() { + s.lock.Lock() + defer s.lock.Unlock() + if _, ok := s.peers[peer.ID()]; ok { + s.trienodeHealIdlers[peer.ID()] = struct{}{} + } + select { + case s.update <- struct{}{}: + default: + } + }() s.lock.Lock() - if _, ok := s.peers[peer.ID()]; ok { - s.trienodeHealIdlers[peer.ID()] = struct{}{} - } - select { - case s.update <- struct{}{}: - default: - } // Ensure the response is for a valid request req, ok := s.trienodeHealReqs[id] if !ok { @@ -2640,10 +2740,12 @@ func (s *Syncer) OnTrieNodes(peer SyncPeer, id uint64, trienodes [][]byte) error // Cross reference the requested trienodes with the response to find gaps // that the serving node is missing - hasher := sha3.NewLegacyKeccak256().(crypto.KeccakState) - hash := make([]byte, 32) - - nodes := make([][]byte, len(req.hashes)) + var ( + hasher = sha3.NewLegacyKeccak256().(crypto.KeccakState) + hash = make([]byte, 32) + nodes = make([][]byte, len(req.hashes)) + fills uint64 + ) for i, j := 0, 0; i < len(trienodes); i++ { // Find the next hash that we've been served, leaving misses with nils hasher.Reset() @@ -2655,20 +2757,26 @@ func (s *Syncer) OnTrieNodes(peer SyncPeer, id uint64, trienodes [][]byte) error } if j < len(req.hashes) { nodes[j] = trienodes[i] + fills++ j++ continue } // We've either ran out of hashes, or got unrequested data logger.Warn("Unexpected healing trienodes", "count", len(trienodes)-i) + // Signal this request as failed, and ready for rescheduling s.scheduleRevertTrienodeHealRequest(req) return errors.New("unexpected healing trienode") } // Response validated, send it to the scheduler for filling + atomic.AddUint64(&s.trienodeHealPend, fills) + defer func() { + atomic.AddUint64(&s.trienodeHealPend, ^(fills - 1)) + }() response := &trienodeHealResponse{ + paths: req.paths, task: req.task, hashes: req.hashes, - paths: req.paths, nodes: nodes, } select { @@ -2692,14 +2800,18 @@ func (s *Syncer) onHealByteCodes(peer SyncPeer, id uint64, bytecodes [][]byte) e // Whether or not the response is valid, we can mark the peer as idle and // notify the scheduler to assign a new task. If the response is invalid, // we'll drop the peer in a bit. + defer func() { + s.lock.Lock() + defer s.lock.Unlock() + if _, ok := s.peers[peer.ID()]; ok { + s.bytecodeHealIdlers[peer.ID()] = struct{}{} + } + select { + case s.update <- struct{}{}: + default: + } + }() s.lock.Lock() - if _, ok := s.peers[peer.ID()]; ok { - s.bytecodeHealIdlers[peer.ID()] = struct{}{} - } - select { - case s.update <- struct{}{}: - default: - } // Ensure the response is for a valid request req, ok := s.bytecodeHealReqs[id] if !ok { @@ -2774,14 +2886,14 @@ func (s *Syncer) onHealByteCodes(peer SyncPeer, id uint64, bytecodes [][]byte) e } // onHealState is a callback method to invoke when a flat state(account -// or storage slot) is downloded during the healing stage. The flat states +// or storage slot) is downloaded during the healing stage. The flat states // can be persisted blindly and can be fixed later in the generation stage. // Note it's not concurrent safe, please handle the concurrent issue outside. func (s *Syncer) onHealState(paths [][]byte, value []byte) error { if len(paths) == 1 { var account types.StateAccount if err := rlp.DecodeBytes(value, &account); err != nil { - return nil + return nil // Returning the error here would drop the remote peer } blob := snapshot.SlimAccountRLP(account.Nonce, account.Balance, account.Root, account.CodeHash) rawdb.WriteAccountSnapshot(s.stateWriter, common.BytesToHash(paths[0]), blob) @@ -2913,8 +3025,9 @@ func (s *capacitySort) Swap(i, j int) { // healRequestSort implements the Sort interface, allowing sorting trienode // heal requests, which is a prerequisite for merging storage-requests. type healRequestSort struct { - hashes []common.Hash - paths []trie.SyncPath + paths []string + hashes []common.Hash + syncPaths []trie.SyncPath } func (t *healRequestSort) Len() int { @@ -2922,8 +3035,8 @@ func (t *healRequestSort) Len() int { } func (t *healRequestSort) Less(i, j int) bool { - a := t.paths[i] - b := t.paths[j] + a := t.syncPaths[i] + b := t.syncPaths[j] switch bytes.Compare(a[0], b[0]) { case -1: return true @@ -2944,8 +3057,9 @@ func (t *healRequestSort) Less(i, j int) bool { } func (t *healRequestSort) Swap(i, j int) { - t.hashes[i], t.hashes[j] = t.hashes[j], t.hashes[i] t.paths[i], t.paths[j] = t.paths[j], t.paths[i] + t.hashes[i], t.hashes[j] = t.hashes[j], t.hashes[i] + t.syncPaths[i], t.syncPaths[j] = t.syncPaths[j], t.syncPaths[i] } // Merge merges the pathsets, so that several storage requests concerning the @@ -2953,7 +3067,7 @@ func (t *healRequestSort) Swap(i, j int) { // OBS: This operation is moot if t has not first been sorted. func (t *healRequestSort) Merge() []TrieNodePathSet { var result []TrieNodePathSet - for _, path := range t.paths { + for _, path := range t.syncPaths { pathset := TrieNodePathSet([][]byte(path)) if len(path) == 1 { // It's an account reference. @@ -2962,7 +3076,7 @@ func (t *healRequestSort) Merge() []TrieNodePathSet { // It's a storage reference. end := len(result) - 1 if len(result) == 0 || !bytes.Equal(pathset[0], result[end][0]) { - // The account doesn't doesn't match last, create a new entry. + // The account doesn't match last, create a new entry. result = append(result, pathset) } else { // It's the same account as the previous one, add to the storage @@ -2976,9 +3090,13 @@ func (t *healRequestSort) Merge() []TrieNodePathSet { // sortByAccountPath takes hashes and paths, and sorts them. After that, it generates // the TrieNodePaths and merges paths which belongs to the same account path. -func sortByAccountPath(hashes []common.Hash, paths []trie.SyncPath) ([]common.Hash, []trie.SyncPath, []TrieNodePathSet) { - n := &healRequestSort{hashes, paths} +func sortByAccountPath(paths []string, hashes []common.Hash) ([]string, []common.Hash, []trie.SyncPath, []TrieNodePathSet) { + var syncPaths []trie.SyncPath + for _, path := range paths { + syncPaths = append(syncPaths, trie.NewSyncPath([]byte(path))) + } + n := &healRequestSort{paths, hashes, syncPaths} sort.Sort(n) pathsets := n.Merge() - return n.hashes, n.paths, pathsets + return n.paths, n.hashes, n.syncPaths, pathsets } diff --git a/eth/protocols/snap/sync_test.go b/eth/protocols/snap/sync_test.go index e727544fa4500..4d9f631b5aab7 100644 --- a/eth/protocols/snap/sync_test.go +++ b/eth/protocols/snap/sync_test.go @@ -1348,9 +1348,11 @@ func getCodeByHash(hash common.Hash) []byte { // makeAccountTrieNoStorage spits out a trie, along with the leafs func makeAccountTrieNoStorage(n int) (*trie.Trie, entrySlice) { - db := trie.NewDatabase(rawdb.NewMemoryDatabase()) - accTrie, _ := trie.New(common.Hash{}, db) - var entries entrySlice + var ( + db = trie.NewDatabase(rawdb.NewMemoryDatabase()) + accTrie = trie.NewEmpty(db) + entries entrySlice + ) for i := uint64(1); i <= uint64(n); i++ { value, _ := rlp.EncodeToBytes(&types.StateAccount{ Nonce: i, @@ -1364,7 +1366,13 @@ func makeAccountTrieNoStorage(n int) (*trie.Trie, entrySlice) { entries = append(entries, elem) } sort.Sort(entries) - accTrie.Commit(nil) + + // Commit the state changes into db and re-create the trie + // for accessing later. + root, nodes, _ := accTrie.Commit(false) + db.Update(trie.NewWithNodeSet(nodes)) + + accTrie, _ = trie.New(common.Hash{}, root, db) return accTrie, entries } @@ -1377,7 +1385,7 @@ func makeBoundaryAccountTrie(n int) (*trie.Trie, entrySlice) { boundaries []common.Hash db = trie.NewDatabase(rawdb.NewMemoryDatabase()) - trie, _ = trie.New(common.Hash{}, db) + accTrie = trie.NewEmpty(db) ) // Initialize boundaries var next common.Hash @@ -1404,7 +1412,7 @@ func makeBoundaryAccountTrie(n int) (*trie.Trie, entrySlice) { CodeHash: getCodeHash(uint64(i)), }) elem := &kv{boundaries[i].Bytes(), value} - trie.Update(elem.k, elem.v) + accTrie.Update(elem.k, elem.v) entries = append(entries, elem) } // Fill other accounts if required @@ -1416,12 +1424,18 @@ func makeBoundaryAccountTrie(n int) (*trie.Trie, entrySlice) { CodeHash: getCodeHash(i), }) elem := &kv{key32(i), value} - trie.Update(elem.k, elem.v) + accTrie.Update(elem.k, elem.v) entries = append(entries, elem) } sort.Sort(entries) - trie.Commit(nil) - return trie, entries + + // Commit the state changes into db and re-create the trie + // for accessing later. + root, nodes, _ := accTrie.Commit(false) + db.Update(trie.NewWithNodeSet(nodes)) + + accTrie, _ = trie.New(common.Hash{}, root, db) + return accTrie, entries } // makeAccountTrieWithStorageWithUniqueStorage creates an account trie where each accounts @@ -1429,10 +1443,12 @@ func makeBoundaryAccountTrie(n int) (*trie.Trie, entrySlice) { func makeAccountTrieWithStorageWithUniqueStorage(accounts, slots int, code bool) (*trie.Trie, entrySlice, map[common.Hash]*trie.Trie, map[common.Hash]entrySlice) { var ( db = trie.NewDatabase(rawdb.NewMemoryDatabase()) - accTrie, _ = trie.New(common.Hash{}, db) + accTrie = trie.NewEmpty(db) entries entrySlice + storageRoots = make(map[common.Hash]common.Hash) storageTries = make(map[common.Hash]*trie.Trie) storageEntries = make(map[common.Hash]entrySlice) + nodes = trie.NewMergedNodeSet() ) // Create n accounts in the trie for i := uint64(1); i <= uint64(accounts); i++ { @@ -1442,9 +1458,9 @@ func makeAccountTrieWithStorageWithUniqueStorage(accounts, slots int, code bool) codehash = getCodeHash(i) } // Create a storage trie - stTrie, stEntries := makeStorageTrieWithSeed(uint64(slots), i, db) - stRoot := stTrie.Hash() - stTrie.Commit(nil) + stRoot, stNodes, stEntries := makeStorageTrieWithSeed(common.BytesToHash(key), uint64(slots), i, db) + nodes.Merge(stNodes) + value, _ := rlp.EncodeToBytes(&types.StateAccount{ Nonce: i, Balance: big.NewInt(int64(i)), @@ -1455,12 +1471,25 @@ func makeAccountTrieWithStorageWithUniqueStorage(accounts, slots int, code bool) accTrie.Update(elem.k, elem.v) entries = append(entries, elem) - storageTries[common.BytesToHash(key)] = stTrie + storageRoots[common.BytesToHash(key)] = stRoot storageEntries[common.BytesToHash(key)] = stEntries } sort.Sort(entries) - accTrie.Commit(nil) + // Commit account trie + root, set, _ := accTrie.Commit(true) + nodes.Merge(set) + + // Commit gathered dirty nodes into database + db.Update(nodes) + + // Re-create tries with new root + accTrie, _ = trie.New(common.Hash{}, root, db) + for i := uint64(1); i <= uint64(accounts); i++ { + key := key32(i) + trie, _ := trie.New(common.BytesToHash(key), storageRoots[common.BytesToHash(key)], db) + storageTries[common.BytesToHash(key)] = trie + } return accTrie, entries, storageTries, storageEntries } @@ -1468,23 +1497,13 @@ func makeAccountTrieWithStorageWithUniqueStorage(accounts, slots int, code bool) func makeAccountTrieWithStorage(accounts, slots int, code, boundary bool) (*trie.Trie, entrySlice, map[common.Hash]*trie.Trie, map[common.Hash]entrySlice) { var ( db = trie.NewDatabase(rawdb.NewMemoryDatabase()) - accTrie, _ = trie.New(common.Hash{}, db) + accTrie = trie.NewEmpty(db) entries entrySlice + storageRoots = make(map[common.Hash]common.Hash) storageTries = make(map[common.Hash]*trie.Trie) storageEntries = make(map[common.Hash]entrySlice) + nodes = trie.NewMergedNodeSet() ) - // Make a storage trie which we reuse for the whole lot - var ( - stTrie *trie.Trie - stEntries entrySlice - ) - if boundary { - stTrie, stEntries = makeBoundaryStorageTrie(slots, db) - } else { - stTrie, stEntries = makeStorageTrieWithSeed(uint64(slots), 0, db) - } - stRoot := stTrie.Hash() - // Create n accounts in the trie for i := uint64(1); i <= uint64(accounts); i++ { key := key32(i) @@ -1492,6 +1511,19 @@ func makeAccountTrieWithStorage(accounts, slots int, code, boundary bool) (*trie if code { codehash = getCodeHash(i) } + // Make a storage trie + var ( + stRoot common.Hash + stNodes *trie.NodeSet + stEntries entrySlice + ) + if boundary { + stRoot, stNodes, stEntries = makeBoundaryStorageTrie(common.BytesToHash(key), slots, db) + } else { + stRoot, stNodes, stEntries = makeStorageTrieWithSeed(common.BytesToHash(key), uint64(slots), 0, db) + } + nodes.Merge(stNodes) + value, _ := rlp.EncodeToBytes(&types.StateAccount{ Nonce: i, Balance: big.NewInt(int64(i)), @@ -1501,21 +1533,41 @@ func makeAccountTrieWithStorage(accounts, slots int, code, boundary bool) (*trie elem := &kv{key, value} accTrie.Update(elem.k, elem.v) entries = append(entries, elem) + // we reuse the same one for all accounts - storageTries[common.BytesToHash(key)] = stTrie + storageRoots[common.BytesToHash(key)] = stRoot storageEntries[common.BytesToHash(key)] = stEntries } sort.Sort(entries) - stTrie.Commit(nil) - accTrie.Commit(nil) + + // Commit account trie + root, set, _ := accTrie.Commit(true) + nodes.Merge(set) + + // Commit gathered dirty nodes into database + db.Update(nodes) + + // Re-create tries with new root + accTrie, err := trie.New(common.Hash{}, root, db) + if err != nil { + panic(err) + } + for i := uint64(1); i <= uint64(accounts); i++ { + key := key32(i) + trie, err := trie.New(common.BytesToHash(key), storageRoots[common.BytesToHash(key)], db) + if err != nil { + panic(err) + } + storageTries[common.BytesToHash(key)] = trie + } return accTrie, entries, storageTries, storageEntries } // makeStorageTrieWithSeed fills a storage trie with n items, returning the // not-yet-committed trie and the sorted entries. The seeds can be used to ensure // that tries are unique. -func makeStorageTrieWithSeed(n, seed uint64, db *trie.Database) (*trie.Trie, entrySlice) { - trie, _ := trie.New(common.Hash{}, db) +func makeStorageTrieWithSeed(owner common.Hash, n, seed uint64, db *trie.Database) (common.Hash, *trie.NodeSet, entrySlice) { + trie, _ := trie.New(owner, common.Hash{}, db) var entries entrySlice for i := uint64(1); i <= n; i++ { // store 'x' at slot 'x' @@ -1530,18 +1582,18 @@ func makeStorageTrieWithSeed(n, seed uint64, db *trie.Database) (*trie.Trie, ent entries = append(entries, elem) } sort.Sort(entries) - trie.Commit(nil) - return trie, entries + root, nodes, _ := trie.Commit(false) + return root, nodes, entries } // makeBoundaryStorageTrie constructs a storage trie. Instead of filling // storage slots normally, this function will fill a few slots which have // boundary hash. -func makeBoundaryStorageTrie(n int, db *trie.Database) (*trie.Trie, entrySlice) { +func makeBoundaryStorageTrie(owner common.Hash, n int, db *trie.Database) (common.Hash, *trie.NodeSet, entrySlice) { var ( entries entrySlice boundaries []common.Hash - trie, _ = trie.New(common.Hash{}, db) + trie, _ = trie.New(owner, common.Hash{}, db) ) // Initialize boundaries var next common.Hash @@ -1581,14 +1633,14 @@ func makeBoundaryStorageTrie(n int, db *trie.Database) (*trie.Trie, entrySlice) entries = append(entries, elem) } sort.Sort(entries) - trie.Commit(nil) - return trie, entries + root, nodes, _ := trie.Commit(false) + return root, nodes, entries } func verifyTrie(db ethdb.KeyValueStore, root common.Hash, t *testing.T) { t.Helper() triedb := trie.NewDatabase(db) - accTrie, err := trie.New(root, triedb) + accTrie, err := trie.New(common.Hash{}, root, triedb) if err != nil { t.Fatal(err) } @@ -1606,7 +1658,7 @@ func verifyTrie(db ethdb.KeyValueStore, root common.Hash, t *testing.T) { } accounts++ if acc.Root != emptyRoot { - storeTrie, err := trie.NewSecure(acc.Root, triedb) + storeTrie, err := trie.NewStateTrie(common.BytesToHash(accIt.Key), acc.Root, triedb) if err != nil { t.Fatal(err) } @@ -1661,7 +1713,7 @@ func TestSyncAccountPerformance(t *testing.T) { // Doing so would bring this number down to zero in this artificial testcase, // but only add extra IO for no reason in practice. if have, want := src.nTrienodeRequests, 1; have != want { - fmt.Printf(src.Stats()) + fmt.Print(src.Stats()) t.Errorf("trie node heal requests wrong, want %d, have %d", want, have) } } diff --git a/eth/state_accessor.go b/eth/state_accessor.go index f01db93a67853..12dba8a0a9b01 100644 --- a/eth/state_accessor.go +++ b/eth/state_accessor.go @@ -44,7 +44,7 @@ import ( // perform Commit or other 'save-to-disk' changes, this should be set to false to avoid // storing trash persistently // - preferDisk: this arg can be used by the caller to signal that even though the 'base' is provided, -// it would be preferrable to start from a fresh state, if we have it on disk. +// it would be preferable to start from a fresh state, if we have it on disk. func (eth *Ethereum) StateAtBlock(block *types.Block, reexec uint64, base *state.StateDB, checkLive bool, preferDisk bool) (statedb *state.StateDB, err error) { var ( current *types.Block diff --git a/eth/sync.go b/eth/sync.go index d67d2311d0d98..8fd86b578cf62 100644 --- a/eth/sync.go +++ b/eth/sync.go @@ -163,7 +163,7 @@ func (cs *chainSyncer) nextSyncOp() *chainSyncOp { // An alternative would be to check the local chain for exceeding the TTD and // avoid triggering a sync in that case, but that could also miss sibling or // other family TTD block being accepted. - if cs.handler.merger.TDDReached() { + if cs.handler.chain.Config().TerminalTotalDifficultyPassed || cs.handler.merger.TDDReached() { return nil } // Ensure we're at minimum peer count. diff --git a/eth/sync_test.go b/eth/sync_test.go index 929a2a9d181c0..0b9f9e1bbaaf7 100644 --- a/eth/sync_test.go +++ b/eth/sync_test.go @@ -30,6 +30,7 @@ import ( // Tests that snap sync is disabled after a successful sync cycle. func TestSnapSyncDisabling66(t *testing.T) { testSnapSyncDisabling(t, eth.ETH66, snap.SNAP1) } +func TestSnapSyncDisabling67(t *testing.T) { testSnapSyncDisabling(t, eth.ETH67, snap.SNAP1) } // Tests that snap sync gets disabled as soon as a real block is successfully // imported into the blockchain. diff --git a/eth/tracers/api.go b/eth/tracers/api.go index 074f193e64ebe..092950e78fa99 100644 --- a/eth/tracers/api.go +++ b/eth/tracers/api.go @@ -20,6 +20,7 @@ import ( "bufio" "bytes" "context" + "encoding/json" "errors" "fmt" "os" @@ -115,7 +116,7 @@ func (context *chainContext) GetHeader(hash common.Hash, number uint64) *types.H return header } -// chainContext construts the context reader which is used by the evm for reading +// chainContext constructs the context reader which is used by the evm for reading // the necessary chain context. func (api *API) chainContext(ctx context.Context) core.ChainContext { return &chainContext{api: api, ctx: ctx} @@ -169,16 +170,17 @@ type TraceConfig struct { Tracer *string Timeout *string Reexec *uint64 + // Config specific to given tracer. Note struct logger + // config are historically embedded in main object. + TracerConfig json.RawMessage } // TraceCallConfig is the config for traceCall API. It holds one more // field to override the state for tracing. type TraceCallConfig struct { - *logger.Config - Tracer *string - Timeout *string - Reexec *uint64 + TraceConfig StateOverrides *ethapi.StateOverride + BlockOverrides *ethapi.BlockOverrides } // StdTraceConfig holds extra parameters to standard-json trace functions. @@ -200,10 +202,10 @@ type blockTraceTask struct { statedb *state.StateDB // Intermediate state prepped for tracing block *types.Block // Block to trace the transactions from rootref common.Hash // Trie root reference held for this task - results []*txTraceResult // Trace results procudes by the task + results []*txTraceResult // Trace results produced by the task } -// blockTraceResult represets the results of tracing a single block when an entire +// blockTraceResult represents the results of tracing a single block when an entire // chain is being traced. type blockTraceResult struct { Block hexutil.Uint64 `json:"block"` // Block number corresponding to this trace @@ -561,7 +563,7 @@ func (api *API) StandardTraceBadBlockToFile(ctx context.Context, hash common.Has // traceBlock configures a new tracer according to the provided configuration, and // executes all the transactions contained within. The return value will be one item -// per transaction, dependent on the requestd tracer. +// per transaction, dependent on the requested tracer. func (api *API) traceBlock(ctx context.Context, block *types.Block, config *TraceConfig) ([]*txTraceResult, error) { if block.NumberU64() == 0 { return nil, errors.New("genesis is not traceable") @@ -705,7 +707,7 @@ func (api *API) standardTraceBlockToFile(ctx context.Context, block *types.Block } } for i, tx := range block.Transactions() { - // Prepare the trasaction for un-traced execution + // Prepare the transaction for un-traced execution var ( msg, _ = tx.AsMessage(signer, block.BaseFee()) txContext = core.NewEVMTxContext(msg) @@ -806,7 +808,6 @@ func (api *API) TraceTransaction(ctx context.Context, hash common.Hash, config * // TraceCall lets you trace a given eth_call. It collects the structured logs // created during the execution of EVM if the given transaction was added on // top of the provided block and returns them as a JSON object. -// You can provide -2 as a block number to trace on top of the pending block. func (api *API) TraceCall(ctx context.Context, args ethapi.TransactionArgs, blockNrOrHash rpc.BlockNumberOrHash, config *TraceCallConfig) (interface{}, error) { // Try to retrieve the specified block var ( @@ -816,6 +817,14 @@ func (api *API) TraceCall(ctx context.Context, args ethapi.TransactionArgs, bloc if hash, ok := blockNrOrHash.Hash(); ok { block, err = api.blockByHash(ctx, hash) } else if number, ok := blockNrOrHash.Number(); ok { + if number == rpc.PendingBlockNumber { + // We don't have access to the miner here. For tracing 'future' transactions, + // it can be done with block- and state-overrides instead, which offers + // more flexibility and stability than trying to trace on 'pending', since + // the contents of 'pending' is unstable and probably not a true representation + // of what the next actual block is likely to contain. + return nil, errors.New("tracing on top of pending is not supported") + } block, err = api.blockByNumber(ctx, number) } else { return nil, errors.New("invalid arguments; neither block nor hash specified") @@ -832,18 +841,19 @@ func (api *API) TraceCall(ctx context.Context, args ethapi.TransactionArgs, bloc if err != nil { return nil, err } - // Apply the customized state rules if required. + vmctx := core.NewEVMBlockContext(block.Header(), api.chainContext(ctx), nil) + // Apply the customization rules if required. if config != nil { if err := config.StateOverrides.Apply(statedb); err != nil { return nil, err } + config.BlockOverrides.Apply(&vmctx) } // Execute the trace msg, err := args.ToMessage(api.backend.RPCGasCap(), block.BaseFee()) if err != nil { return nil, err } - vmctx := core.NewEVMBlockContext(block.Header(), api.chainContext(ctx), nil) var traceConfig *TraceConfig if config != nil { @@ -873,7 +883,7 @@ func (api *API) traceTx(ctx context.Context, message core.Message, txctx *Contex // Default tracer is the struct logger tracer = logger.NewStructLogger(config.Config) if config.Tracer != nil { - tracer, err = New(*config.Tracer, txctx) + tracer, err = New(*config.Tracer, txctx, config.TracerConfig) if err != nil { return nil, err } @@ -909,9 +919,7 @@ func APIs(backend Backend) []rpc.API { return []rpc.API{ { Namespace: "debug", - Version: "1.0", Service: NewAPI(backend), - Public: false, }, } } diff --git a/eth/tracers/api_test.go b/eth/tracers/api_test.go index af41f05d212b8..bc12b9275160e 100644 --- a/eth/tracers/api_test.go +++ b/eth/tracers/api_test.go @@ -196,13 +196,12 @@ func TestTraceCall(t *testing.T) { tx, _ := types.SignTx(types.NewTransaction(uint64(i), accounts[1].addr, big.NewInt(1000), params.TxGas, b.BaseFee(), nil), signer, accounts[0].key) b.AddTx(tx) })) - var testSuite = []struct { blockNumber rpc.BlockNumber call ethapi.TransactionArgs config *TraceCallConfig expectErr error - expect interface{} + expect string }{ // Standard JSON trace upon the genesis, plain transfer. { @@ -214,12 +213,7 @@ func TestTraceCall(t *testing.T) { }, config: nil, expectErr: nil, - expect: &logger.ExecutionResult{ - Gas: params.TxGas, - Failed: false, - ReturnValue: "", - StructLogs: []logger.StructLogRes{}, - }, + expect: `{"gas":21000,"failed":false,"returnValue":"","structLogs":[]}`, }, // Standard JSON trace upon the head, plain transfer. { @@ -231,12 +225,7 @@ func TestTraceCall(t *testing.T) { }, config: nil, expectErr: nil, - expect: &logger.ExecutionResult{ - Gas: params.TxGas, - Failed: false, - ReturnValue: "", - StructLogs: []logger.StructLogRes{}, - }, + expect: `{"gas":21000,"failed":false,"returnValue":"","structLogs":[]}`, }, // Standard JSON trace upon the non-existent block, error expects { @@ -248,7 +237,7 @@ func TestTraceCall(t *testing.T) { }, config: nil, expectErr: fmt.Errorf("block #%d not found", genBlocks+1), - expect: nil, + //expect: nil, }, // Standard JSON trace upon the latest block { @@ -260,14 +249,9 @@ func TestTraceCall(t *testing.T) { }, config: nil, expectErr: nil, - expect: &logger.ExecutionResult{ - Gas: params.TxGas, - Failed: false, - ReturnValue: "", - StructLogs: []logger.StructLogRes{}, - }, + expect: `{"gas":21000,"failed":false,"returnValue":"","structLogs":[]}`, }, - // Standard JSON trace upon the pending block + // Tracing on 'pending' should fail: { blockNumber: rpc.PendingBlockNumber, call: ethapi.TransactionArgs{ @@ -276,36 +260,48 @@ func TestTraceCall(t *testing.T) { Value: (*hexutil.Big)(big.NewInt(1000)), }, config: nil, - expectErr: nil, - expect: &logger.ExecutionResult{ - Gas: params.TxGas, - Failed: false, - ReturnValue: "", - StructLogs: []logger.StructLogRes{}, + expectErr: errors.New("tracing on top of pending is not supported"), + }, + { + blockNumber: rpc.LatestBlockNumber, + call: ethapi.TransactionArgs{ + From: &accounts[0].addr, + Input: &hexutil.Bytes{0x43}, // blocknumber + }, + config: &TraceCallConfig{ + BlockOverrides: ðapi.BlockOverrides{Number: (*hexutil.Big)(big.NewInt(0x1337))}, }, + expectErr: nil, + expect: ` {"gas":53018,"failed":false,"returnValue":"","structLogs":[ + {"pc":0,"op":"NUMBER","gas":24946984,"gasCost":2,"depth":1,"stack":[]}, + {"pc":1,"op":"STOP","gas":24946982,"gasCost":0,"depth":1,"stack":["0x1337"]}]}`, }, } - for _, testspec := range testSuite { + for i, testspec := range testSuite { result, err := api.TraceCall(context.Background(), testspec.call, rpc.BlockNumberOrHash{BlockNumber: &testspec.blockNumber}, testspec.config) if testspec.expectErr != nil { if err == nil { - t.Errorf("Expect error %v, get nothing", testspec.expectErr) + t.Errorf("test %d: expect error %v, got nothing", i, testspec.expectErr) continue } if !reflect.DeepEqual(err, testspec.expectErr) { - t.Errorf("Error mismatch, want %v, get %v", testspec.expectErr, err) + t.Errorf("test %d: error mismatch, want %v, git %v", i, testspec.expectErr, err) } } else { if err != nil { - t.Errorf("Expect no error, get %v", err) + t.Errorf("test %d: expect no error, got %v", i, err) continue } var have *logger.ExecutionResult if err := json.Unmarshal(result.(json.RawMessage), &have); err != nil { - t.Errorf("failed to unmarshal result %v", err) + t.Errorf("test %d: failed to unmarshal result %v", i, err) + } + var want *logger.ExecutionResult + if err := json.Unmarshal([]byte(testspec.expect), &want); err != nil { + t.Errorf("test %d: failed to unmarshal result %v", i, err) } - if !reflect.DeepEqual(have, testspec.expect) { - t.Errorf("Result mismatch, want %v, get %v", testspec.expect, have) + if !reflect.DeepEqual(have, want) { + t.Errorf("test %d: result mismatch, want %v, got %v", i, testspec.expect, string(result.(json.RawMessage))) } } } @@ -446,7 +442,7 @@ func TestTracingWithOverrides(t *testing.T) { type res struct { Gas int Failed bool - returnValue string + ReturnValue string } var testSuite = []struct { blockNumber rpc.BlockNumber @@ -457,7 +453,7 @@ func TestTracingWithOverrides(t *testing.T) { }{ // Call which can only succeed if state is state overridden { - blockNumber: rpc.PendingBlockNumber, + blockNumber: rpc.LatestBlockNumber, call: ethapi.TransactionArgs{ From: &randomAccounts[0].addr, To: &randomAccounts[1].addr, @@ -472,7 +468,7 @@ func TestTracingWithOverrides(t *testing.T) { }, // Invalid call without state overriding { - blockNumber: rpc.PendingBlockNumber, + blockNumber: rpc.LatestBlockNumber, call: ethapi.TransactionArgs{ From: &randomAccounts[0].addr, To: &randomAccounts[1].addr, @@ -498,7 +494,7 @@ func TestTracingWithOverrides(t *testing.T) { // } // } { - blockNumber: rpc.PendingBlockNumber, + blockNumber: rpc.LatestBlockNumber, call: ethapi.TransactionArgs{ From: &randomAccounts[0].addr, To: &randomAccounts[2].addr, @@ -515,6 +511,39 @@ func TestTracingWithOverrides(t *testing.T) { }, want: `{"gas":23347,"failed":false,"returnValue":"000000000000000000000000000000000000000000000000000000000000007b"}`, }, + { // Override blocknumber + blockNumber: rpc.LatestBlockNumber, + call: ethapi.TransactionArgs{ + From: &accounts[0].addr, + // BLOCKNUMBER PUSH1 MSTORE + Input: newRPCBytes(common.Hex2Bytes("4360005260206000f3")), + //&hexutil.Bytes{0x43}, // blocknumber + }, + config: &TraceCallConfig{ + BlockOverrides: ðapi.BlockOverrides{Number: (*hexutil.Big)(big.NewInt(0x1337))}, + }, + want: `{"gas":59537,"failed":false,"returnValue":"0000000000000000000000000000000000000000000000000000000000001337"}`, + }, + { // Override blocknumber, and query a blockhash + blockNumber: rpc.LatestBlockNumber, + call: ethapi.TransactionArgs{ + From: &accounts[0].addr, + Input: &hexutil.Bytes{ + 0x60, 0x00, 0x40, // BLOCKHASH(0) + 0x60, 0x00, 0x52, // STORE memory offset 0 + 0x61, 0x13, 0x36, 0x40, // BLOCKHASH(0x1336) + 0x60, 0x20, 0x52, // STORE memory offset 32 + 0x61, 0x13, 0x37, 0x40, // BLOCKHASH(0x1337) + 0x60, 0x40, 0x52, // STORE memory offset 64 + 0x60, 0x60, 0x60, 0x00, 0xf3, // RETURN (0-96) + + }, // blocknumber + }, + config: &TraceCallConfig{ + BlockOverrides: ðapi.BlockOverrides{Number: (*hexutil.Big)(big.NewInt(0x1337))}, + }, + want: `{"gas":72666,"failed":false,"returnValue":"000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"}`, + }, } for i, tc := range testSuite { result, err := api.TraceCall(context.Background(), tc.call, rpc.BlockNumberOrHash{BlockNumber: &tc.blockNumber}, tc.config) diff --git a/eth/tracers/internal/tracetest/calltrace_test.go b/eth/tracers/internal/tracetest/calltrace_test.go index cbf20ed00c0ce..d2c50656d9a86 100644 --- a/eth/tracers/internal/tracetest/calltrace_test.go +++ b/eth/tracers/internal/tracetest/calltrace_test.go @@ -39,7 +39,7 @@ import ( "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/tests" - // Force-load native and js pacakges, to trigger registration + // Force-load native and js packages, to trigger registration _ "github.com/ethereum/go-ethereum/eth/tracers/js" _ "github.com/ethereum/go-ethereum/eth/tracers/native" ) @@ -118,10 +118,11 @@ type callTrace struct { // callTracerTest defines a single test to check the call tracer against. type callTracerTest struct { - Genesis *core.Genesis `json:"genesis"` - Context *callContext `json:"context"` - Input string `json:"input"` - Result *callTrace `json:"result"` + Genesis *core.Genesis `json:"genesis"` + Context *callContext `json:"context"` + Input string `json:"input"` + TracerConfig json.RawMessage `json:"tracerConfig"` + Result *callTrace `json:"result"` } // Iterates over all the input-output datasets in the tracer test harness and @@ -179,7 +180,7 @@ func testCallTracer(tracerName string, dirPath string, t *testing.T) { } _, statedb = tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false) ) - tracer, err := tracers.New(tracerName, new(tracers.Context)) + tracer, err := tracers.New(tracerName, new(tracers.Context), test.TracerConfig) if err != nil { t.Fatalf("failed to create call tracer: %v", err) } @@ -293,7 +294,7 @@ func benchTracer(tracerName string, test *callTracerTest, b *testing.B) { b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { - tracer, err := tracers.New(tracerName, new(tracers.Context)) + tracer, err := tracers.New(tracerName, new(tracers.Context), nil) if err != nil { b.Fatalf("failed to create call tracer: %v", err) } @@ -359,7 +360,7 @@ func TestZeroValueToNotExitCall(t *testing.T) { } _, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), alloc, false) // Create the tracer, the EVM environment and run it - tracer, err := tracers.New("callTracer", nil) + tracer, err := tracers.New("callTracer", nil, nil) if err != nil { t.Fatalf("failed to create call tracer: %v", err) } diff --git a/eth/tracers/internal/tracetest/testdata/call_tracer/simple_onlytop.json b/eth/tracers/internal/tracetest/testdata/call_tracer/simple_onlytop.json new file mode 100644 index 0000000000000..ac1fef44098e7 --- /dev/null +++ b/eth/tracers/internal/tracetest/testdata/call_tracer/simple_onlytop.json @@ -0,0 +1,72 @@ +{ + "context": { + "difficulty": "3502894804", + "gasLimit": "4722976", + "miner": "0x1585936b53834b021f68cc13eeefdec2efc8e724", + "number": "2289806", + "timestamp": "1513601314" + }, + "genesis": { + "alloc": { + "0x0024f658a46fbb89d8ac105e98d7ac7cbbaf27c5": { + "balance": "0x0", + "code": "0x", + "nonce": "22", + "storage": {} + }, + "0x3b873a919aa0512d5a0f09e6dcceaa4a6727fafe": { + "balance": "0x4d87094125a369d9bd5", + "code": "0x606060405236156100935763ffffffff60e060020a60003504166311ee8382811461009c57806313af4035146100be5780631f5e8f4c146100ee57806324daddc5146101125780634921a91a1461013b57806363e4bff414610157578063764978f91461017f578063893d20e8146101a1578063ba40aaa1146101cd578063cebc9a82146101f4578063e177246e14610216575b61009a5b5b565b005b34156100a457fe5b6100ac61023d565b60408051918252519081900360200190f35b34156100c657fe5b6100da600160a060020a0360043516610244565b604080519115158252519081900360200190f35b34156100f657fe5b6100da610307565b604080519115158252519081900360200190f35b341561011a57fe5b6100da6004351515610318565b604080519115158252519081900360200190f35b6100da6103d6565b604080519115158252519081900360200190f35b6100da600160a060020a0360043516610420565b604080519115158252519081900360200190f35b341561018757fe5b6100ac61046c565b60408051918252519081900360200190f35b34156101a957fe5b6101b1610473565b60408051600160a060020a039092168252519081900360200190f35b34156101d557fe5b6100da600435610483565b604080519115158252519081900360200190f35b34156101fc57fe5b6100ac61050d565b60408051918252519081900360200190f35b341561021e57fe5b6100da600435610514565b604080519115158252519081900360200190f35b6003545b90565b60006000610250610473565b600160a060020a031633600160a060020a03161415156102705760006000fd5b600160a060020a03831615156102865760006000fd5b50600054600160a060020a0390811690831681146102fb57604051600160a060020a0380851691908316907ffcf23a92150d56e85e3a3d33b357493246e55783095eb6a733eb8439ffc752c890600090a360008054600160a060020a031916600160a060020a03851617905560019150610300565b600091505b5b50919050565b60005460a060020a900460ff165b90565b60006000610324610473565b600160a060020a031633600160a060020a03161415156103445760006000fd5b5060005460a060020a900460ff16801515831515146102fb576000546040805160a060020a90920460ff1615158252841515602083015280517fe6cd46a119083b86efc6884b970bfa30c1708f53ba57b86716f15b2f4551a9539281900390910190a16000805460a060020a60ff02191660a060020a8515150217905560019150610300565b600091505b5b50919050565b60006103e0610307565b801561040557506103ef610473565b600160a060020a031633600160a060020a031614155b156104105760006000fd5b610419336105a0565b90505b5b90565b600061042a610307565b801561044f5750610439610473565b600160a060020a031633600160a060020a031614155b1561045a5760006000fd5b610463826105a0565b90505b5b919050565b6001545b90565b600054600160a060020a03165b90565b6000600061048f610473565b600160a060020a031633600160a060020a03161415156104af5760006000fd5b506001548281146102fb57604080518281526020810185905281517f79a3746dde45672c9e8ab3644b8bb9c399a103da2dc94b56ba09777330a83509929181900390910190a160018381559150610300565b600091505b5b50919050565b6002545b90565b60006000610520610473565b600160a060020a031633600160a060020a03161415156105405760006000fd5b506002548281146102fb57604080518281526020810185905281517ff6991a728965fedd6e927fdf16bdad42d8995970b4b31b8a2bf88767516e2494929181900390910190a1600283905560019150610300565b600091505b5b50919050565b60006000426105ad61023d565b116102fb576105c46105bd61050d565b4201610652565b6105cc61046c565b604051909150600160a060020a038416908290600081818185876187965a03f1925050501561063d57604080518281529051600160a060020a038516917f9bca65ce52fdef8a470977b51f247a2295123a4807dfa9e502edf0d30722da3b919081900360200190a260019150610300565b6102fb42610652565b5b600091505b50919050565b60038190555b505600a165627a7a72305820f3c973c8b7ed1f62000b6701bd5b708469e19d0f1d73fde378a56c07fd0b19090029", + "nonce": "1", + "storage": { + "0x0000000000000000000000000000000000000000000000000000000000000000": "0x000000000000000000000001b436ba50d378d4bbc8660d312a13df6af6e89dfb", + "0x0000000000000000000000000000000000000000000000000000000000000001": "0x00000000000000000000000000000000000000000000000006f05b59d3b20000", + "0x0000000000000000000000000000000000000000000000000000000000000002": "0x000000000000000000000000000000000000000000000000000000000000003c", + "0x0000000000000000000000000000000000000000000000000000000000000003": "0x000000000000000000000000000000000000000000000000000000005a37b834" + } + }, + "0xb436ba50d378d4bbc8660d312a13df6af6e89dfb": { + "balance": "0x1780d77678137ac1b775", + "code": "0x", + "nonce": "29072", + "storage": {} + } + }, + "config": { + "byzantiumBlock": 1700000, + "chainId": 3, + "daoForkSupport": true, + "eip150Block": 0, + "eip150Hash": "0x41941023680923e0fe4d74a34bdac8141f2540e3ae90623718e47d66d1ca4a2d", + "eip155Block": 10, + "eip158Block": 10, + "ethash": {}, + "homesteadBlock": 0 + }, + "difficulty": "3509749784", + "extraData": "0x4554482e45544846414e532e4f52472d4641313738394444", + "gasLimit": "4727564", + "hash": "0x609948ac3bd3c00b7736b933248891d6c901ee28f066241bddb28f4e00a9f440", + "miner": "0xbbf5029fd710d227630c8b7d338051b8e76d50b3", + "mixHash": "0xb131e4507c93c7377de00e7c271bf409ec7492767142ff0f45c882f8068c2ada", + "nonce": "0x4eb12e19c16d43da", + "number": "2289805", + "stateRoot": "0xc7f10f352bff82fac3c2999d3085093d12652e19c7fd32591de49dc5d91b4f1f", + "timestamp": "1513601261", + "totalDifficulty": "7143276353481064" + }, + "input": "0xf88b8271908506fc23ac0083015f90943b873a919aa0512d5a0f09e6dcceaa4a6727fafe80a463e4bff40000000000000000000000000024f658a46fbb89d8ac105e98d7ac7cbbaf27c52aa0bdce0b59e8761854e857fe64015f06dd08a4fbb7624f6094893a79a72e6ad6bea01d9dde033cff7bb235a3163f348a6d7ab8d6b52bc0963a95b91612e40ca766a4", + "tracerConfig": { + "onlyTopCall": true + }, + "result": { + "from": "0xb436ba50d378d4bbc8660d312a13df6af6e89dfb", + "gas": "0x10738", + "gasUsed": "0x3ef9", + "input": "0x63e4bff40000000000000000000000000024f658a46fbb89d8ac105e98d7ac7cbbaf27c5", + "output": "0x0000000000000000000000000000000000000000000000000000000000000001", + "to": "0x3b873a919aa0512d5a0f09e6dcceaa4a6727fafe", + "type": "CALL", + "value": "0x0" + } +} diff --git a/eth/tracers/js/goja.go b/eth/tracers/js/goja.go index f0c78c084bd95..8238bb603dd30 100644 --- a/eth/tracers/js/goja.go +++ b/eth/tracers/js/goja.go @@ -55,11 +55,7 @@ type fromBufFn = func(vm *goja.Runtime, buf goja.Value, allowString bool) ([]byt func toBuf(vm *goja.Runtime, bufType goja.Value, val []byte) (goja.Value, error) { // bufType is usually Uint8Array. This is equivalent to `new Uint8Array(val)` in JS. - res, err := vm.New(bufType, vm.ToValue(val)) - if err != nil { - return nil, err - } - return vm.ToValue(res), nil + return vm.New(bufType, vm.ToValue(vm.NewArrayBuffer(val))) } func fromBuf(vm *goja.Runtime, bufType goja.Value, buf goja.Value, allowString bool) ([]byte, error) { @@ -70,6 +66,7 @@ func fromBuf(vm *goja.Runtime, bufType goja.Value, buf goja.Value, allowString b break } return common.FromHex(obj.String()), nil + case "Array": var b []byte if err := vm.ExportTo(buf, &b); err != nil { @@ -81,10 +78,7 @@ func fromBuf(vm *goja.Runtime, bufType goja.Value, buf goja.Value, allowString b if !obj.Get("constructor").SameAs(bufType) { break } - var b []byte - if err := vm.ExportTo(buf, &b); err != nil { - return nil, err - } + b := obj.Get("buffer").Export().(goja.ArrayBuffer).Bytes() return b, nil } return nil, fmt.Errorf("invalid buffer type") @@ -131,7 +125,7 @@ type jsTracer struct { // The methods `result` and `fault` are required to be present. // The methods `step`, `enter`, and `exit` are optional, but note that // `enter` and `exit` always go together. -func newJsTracer(code string, ctx *tracers.Context) (tracers.Tracer, error) { +func newJsTracer(code string, ctx *tracers.Context, cfg json.RawMessage) (tracers.Tracer, error) { if c, ok := assetTracers[code]; ok { code = c } @@ -183,6 +177,17 @@ func newJsTracer(code string, ctx *tracers.Context) (tracers.Tracer, error) { t.exit = exit t.result = result t.fault = fault + + // Pass in config + if setup, ok := goja.AssertFunction(obj.Get("setup")); ok { + cfgStr := "{}" + if cfg != nil { + cfgStr = string(cfg) + } + if _, err := setup(obj, vm.ToValue(cfgStr)); err != nil { + return nil, err + } + } // Setup objects carrying data to JS. These are created once and re-used. t.log = &steplog{ vm: vm, @@ -554,10 +559,10 @@ func (mo *memoryObj) slice(begin, end int64) ([]byte, error) { return []byte{}, nil } if end < begin || begin < 0 { - return nil, fmt.Errorf("Tracer accessed out of bound memory: offset %d, end %d", begin, end) + return nil, fmt.Errorf("tracer accessed out of bound memory: offset %d, end %d", begin, end) } if mo.memory.Len() < int(end) { - return nil, fmt.Errorf("Tracer accessed out of bound memory: available %d, offset %d, size %d", mo.memory.Len(), begin, end-begin) + return nil, fmt.Errorf("tracer accessed out of bound memory: available %d, offset %d, size %d", mo.memory.Len(), begin, end-begin) } return mo.memory.GetCopy(begin, end-begin), nil } @@ -579,7 +584,7 @@ func (mo *memoryObj) GetUint(addr int64) goja.Value { // getUint returns the 32 bytes at the specified address interpreted as a uint. func (mo *memoryObj) getUint(addr int64) (*big.Int, error) { if mo.memory.Len() < int(addr)+32 || addr < 0 { - return nil, fmt.Errorf("Tracer accessed out of bound memory: available %d, offset %d, size %d", mo.memory.Len(), addr, 32) + return nil, fmt.Errorf("tracer accessed out of bound memory: available %d, offset %d, size %d", mo.memory.Len(), addr, 32) } return new(big.Int).SetBytes(mo.memory.GetPtr(addr, 32)), nil } @@ -619,7 +624,7 @@ func (s *stackObj) Peek(idx int) goja.Value { // peek returns the nth-from-the-top element of the stack. func (s *stackObj) peek(idx int) (*big.Int, error) { if len(s.stack.Data()) <= idx || idx < 0 { - return nil, fmt.Errorf("Tracer accessed out of bound stack: size %d, index %d", len(s.stack.Data()), idx) + return nil, fmt.Errorf("tracer accessed out of bound stack: size %d, index %d", len(s.stack.Data()), idx) } return s.stack.Back(idx).ToBig(), nil } @@ -765,7 +770,7 @@ func (co *contractObj) GetValue() goja.Value { } func (co *contractObj) GetInput() goja.Value { - input := co.contract.Input + input := common.CopyBytes(co.contract.Input) res, err := co.toBuf(co.vm, input) if err != nil { co.vm.Interrupt(err) @@ -884,7 +889,6 @@ func (r *callframeResult) GetError() goja.Value { return r.vm.ToValue(r.err.Error()) } return goja.Undefined() - } func (r *callframeResult) setupObject() *goja.Object { diff --git a/eth/tracers/js/internal/tracers/4byte_tracer_legacy.js b/eth/tracers/js/internal/tracers/4byte_tracer_legacy.js index 462b4ad4cb550..e4714b8bfb762 100644 --- a/eth/tracers/js/internal/tracers/4byte_tracer_legacy.js +++ b/eth/tracers/js/internal/tracers/4byte_tracer_legacy.js @@ -46,7 +46,7 @@ return false; }, - // store save the given indentifier and datasize. + // store save the given identifier and datasize. store: function(id, size){ var key = "" + toHex(id) + "-" + size; this.ids[key] = this.ids[key] + 1 || 1; diff --git a/eth/tracers/js/tracer_test.go b/eth/tracers/js/tracer_test.go index 2863bd4451b8a..80a002d5af28e 100644 --- a/eth/tracers/js/tracer_test.go +++ b/eth/tracers/js/tracer_test.go @@ -85,7 +85,7 @@ func runTrace(tracer tracers.Tracer, vmctx *vmContext, chaincfg *params.ChainCon func TestTracer(t *testing.T) { execTracer := func(code string) ([]byte, string) { t.Helper() - tracer, err := newJsTracer(code, nil) + tracer, err := newJsTracer(code, nil, nil) if err != nil { t.Fatal(err) } @@ -103,15 +103,15 @@ func TestTracer(t *testing.T) { { // tests that we don't panic on bad arguments to memory access code: "{depths: [], step: function(log) { this.depths.push(log.memory.slice(-1,-2)); }, fault: function() {}, result: function() { return this.depths; }}", want: ``, - fail: "Tracer accessed out of bound memory: offset -1, end -2 at step (:1:53(15)) in server-side tracer function 'step'", + fail: "tracer accessed out of bound memory: offset -1, end -2 at step (:1:53(15)) in server-side tracer function 'step'", }, { // tests that we don't panic on bad arguments to stack peeks code: "{depths: [], step: function(log) { this.depths.push(log.stack.peek(-1)); }, fault: function() {}, result: function() { return this.depths; }}", want: ``, - fail: "Tracer accessed out of bound stack: size 0, index -1 at step (:1:53(13)) in server-side tracer function 'step'", + fail: "tracer accessed out of bound stack: size 0, index -1 at step (:1:53(13)) in server-side tracer function 'step'", }, { // tests that we don't panic on bad arguments to memory getUint code: "{ depths: [], step: function(log, db) { this.depths.push(log.memory.getUint(-64));}, fault: function() {}, result: function() { return this.depths; }}", want: ``, - fail: "Tracer accessed out of bound memory: available 0, offset -64, size 32 at step (:1:58(13)) in server-side tracer function 'step'", + fail: "tracer accessed out of bound memory: available 0, offset -64, size 32 at step (:1:58(13)) in server-side tracer function 'step'", }, { // tests some general counting code: "{count: 0, step: function() { this.count += 1; }, fault: function() {}, result: function() { return this.count; }}", want: `3`, @@ -149,7 +149,7 @@ func TestTracer(t *testing.T) { func TestHalt(t *testing.T) { timeout := errors.New("stahp") - tracer, err := newJsTracer("{step: function() { while(1); }, result: function() { return null; }, fault: function(){}}", nil) + tracer, err := newJsTracer("{step: function() { while(1); }, result: function() { return null; }, fault: function(){}}", nil, nil) if err != nil { t.Fatal(err) } @@ -163,7 +163,7 @@ func TestHalt(t *testing.T) { } func TestHaltBetweenSteps(t *testing.T) { - tracer, err := newJsTracer("{step: function() {}, fault: function() {}, result: function() { return null; }}", nil) + tracer, err := newJsTracer("{step: function() {}, fault: function() {}, result: function() { return null; }}", nil, nil) if err != nil { t.Fatal(err) } @@ -187,7 +187,7 @@ func TestHaltBetweenSteps(t *testing.T) { func TestNoStepExec(t *testing.T) { execTracer := func(code string) []byte { t.Helper() - tracer, err := newJsTracer(code, nil) + tracer, err := newJsTracer(code, nil, nil) if err != nil { t.Fatal(err) } @@ -221,7 +221,7 @@ func TestIsPrecompile(t *testing.T) { chaincfg.IstanbulBlock = big.NewInt(200) chaincfg.BerlinBlock = big.NewInt(300) txCtx := vm.TxContext{GasPrice: big.NewInt(100000)} - tracer, err := newJsTracer("{addr: toAddress('0000000000000000000000000000000000000009'), res: null, step: function() { this.res = isPrecompiled(this.addr); }, fault: function() {}, result: function() { return this.res; }}", nil) + tracer, err := newJsTracer("{addr: toAddress('0000000000000000000000000000000000000009'), res: null, step: function() { this.res = isPrecompiled(this.addr); }, fault: function() {}, result: function() { return this.res; }}", nil, nil) if err != nil { t.Fatal(err) } @@ -232,30 +232,30 @@ func TestIsPrecompile(t *testing.T) { t.Error(err) } if string(res) != "false" { - t.Errorf("Tracer should not consider blake2f as precompile in byzantium") + t.Errorf("tracer should not consider blake2f as precompile in byzantium") } - tracer, _ = newJsTracer("{addr: toAddress('0000000000000000000000000000000000000009'), res: null, step: function() { this.res = isPrecompiled(this.addr); }, fault: function() {}, result: function() { return this.res; }}", nil) + tracer, _ = newJsTracer("{addr: toAddress('0000000000000000000000000000000000000009'), res: null, step: function() { this.res = isPrecompiled(this.addr); }, fault: function() {}, result: function() { return this.res; }}", nil, nil) blockCtx = vm.BlockContext{BlockNumber: big.NewInt(250)} res, err = runTrace(tracer, &vmContext{blockCtx, txCtx}, chaincfg) if err != nil { t.Error(err) } if string(res) != "true" { - t.Errorf("Tracer should consider blake2f as precompile in istanbul") + t.Errorf("tracer should consider blake2f as precompile in istanbul") } } func TestEnterExit(t *testing.T) { // test that either both or none of enter() and exit() are defined - if _, err := newJsTracer("{step: function() {}, fault: function() {}, result: function() { return null; }, enter: function() {}}", new(tracers.Context)); err == nil { + if _, err := newJsTracer("{step: function() {}, fault: function() {}, result: function() { return null; }, enter: function() {}}", new(tracers.Context), nil); err == nil { t.Fatal("tracer creation should've failed without exit() definition") } - if _, err := newJsTracer("{step: function() {}, fault: function() {}, result: function() { return null; }, enter: function() {}, exit: function() {}}", new(tracers.Context)); err != nil { + if _, err := newJsTracer("{step: function() {}, fault: function() {}, result: function() { return null; }, enter: function() {}, exit: function() {}}", new(tracers.Context), nil); err != nil { t.Fatal(err) } // test that the enter and exit method are correctly invoked and the values passed - tracer, err := newJsTracer("{enters: 0, exits: 0, enterGas: 0, gasUsed: 0, step: function() {}, fault: function() {}, result: function() { return {enters: this.enters, exits: this.exits, enterGas: this.enterGas, gasUsed: this.gasUsed} }, enter: function(frame) { this.enters++; this.enterGas = frame.getGas(); }, exit: function(res) { this.exits++; this.gasUsed = res.getGasUsed(); }}", new(tracers.Context)) + tracer, err := newJsTracer("{enters: 0, exits: 0, enterGas: 0, gasUsed: 0, step: function() {}, fault: function() {}, result: function() { return {enters: this.enters, exits: this.exits, enterGas: this.enterGas, gasUsed: this.gasUsed} }, enter: function(frame) { this.enters++; this.enterGas = frame.getGas(); }, exit: function(res) { this.exits++; this.gasUsed = res.getGasUsed(); }}", new(tracers.Context), nil) if err != nil { t.Fatal(err) } @@ -274,3 +274,33 @@ func TestEnterExit(t *testing.T) { t.Errorf("Number of invocations of enter() and exit() is wrong. Have %s, want %s\n", have, want) } } + +func TestSetup(t *testing.T) { + // Test empty config + _, err := newJsTracer(`{setup: function(cfg) { if (cfg !== "{}") { throw("invalid empty config") } }, fault: function() {}, result: function() {}}`, new(tracers.Context), nil) + if err != nil { + t.Error(err) + } + + cfg, err := json.Marshal(map[string]string{"foo": "bar"}) + if err != nil { + t.Fatal(err) + } + // Test no setup func + _, err = newJsTracer(`{fault: function() {}, result: function() {}}`, new(tracers.Context), cfg) + if err != nil { + t.Fatal(err) + } + // Test config value + tracer, err := newJsTracer("{config: null, setup: function(cfg) { this.config = JSON.parse(cfg) }, step: function() {}, fault: function() {}, result: function() { return this.config.foo }}", new(tracers.Context), cfg) + if err != nil { + t.Fatal(err) + } + have, err := tracer.GetResult() + if err != nil { + t.Fatal(err) + } + if string(have) != `"bar"` { + t.Errorf("tracer returned wrong result. have: %s, want: \"bar\"\n", string(have)) + } +} diff --git a/eth/tracers/logger/logger.go b/eth/tracers/logger/logger.go index fe850d6b3e61c..07aa2f2b4301c 100644 --- a/eth/tracers/logger/logger.go +++ b/eth/tracers/logger/logger.go @@ -40,7 +40,7 @@ type Storage map[common.Hash]common.Hash // Copy duplicates the current storage. func (s Storage) Copy() Storage { - cpy := make(Storage) + cpy := make(Storage, len(s)) for key, value := range s { cpy[key] = value } @@ -224,7 +224,7 @@ func (l *StructLogger) CaptureEnd(output []byte, gasUsed uint64, t time.Duration l.output = output l.err = err if l.cfg.Debug { - fmt.Printf("0x%x\n", output) + fmt.Printf("%#x\n", output) if err != nil { fmt.Printf(" error: %v\n", err) } @@ -346,11 +346,11 @@ func NewMarkdownLogger(cfg *Config, writer io.Writer) *mdLogger { func (t *mdLogger) CaptureStart(env *vm.EVM, from common.Address, to common.Address, create bool, input []byte, gas uint64, value *big.Int) { t.env = env if !create { - fmt.Fprintf(t.out, "From: `%v`\nTo: `%v`\nData: `0x%x`\nGas: `%d`\nValue `%v` wei\n", + fmt.Fprintf(t.out, "From: `%v`\nTo: `%v`\nData: `%#x`\nGas: `%d`\nValue `%v` wei\n", from.String(), to.String(), input, gas, value) } else { - fmt.Fprintf(t.out, "From: `%v`\nCreate at: `%v`\nData: `0x%x`\nGas: `%d`\nValue `%v` wei\n", + fmt.Fprintf(t.out, "From: `%v`\nCreate at: `%v`\nData: `%#x`\nGas: `%d`\nValue `%v` wei\n", from.String(), to.String(), input, gas, value) } @@ -387,7 +387,7 @@ func (t *mdLogger) CaptureFault(pc uint64, op vm.OpCode, gas, cost uint64, scope } func (t *mdLogger) CaptureEnd(output []byte, gasUsed uint64, tm time.Duration, err error) { - fmt.Fprintf(t.out, "\nOutput: `0x%x`\nConsumed gas: `%d`\nError: `%v`\n", + fmt.Fprintf(t.out, "\nOutput: `%#x`\nConsumed gas: `%d`\nError: `%v`\n", output, gasUsed, err) } diff --git a/eth/tracers/native/4byte.go b/eth/tracers/native/4byte.go index 92cc70994c323..34e608bfd60d6 100644 --- a/eth/tracers/native/4byte.go +++ b/eth/tracers/native/4byte.go @@ -55,11 +55,11 @@ type fourByteTracer struct { // newFourByteTracer returns a native go tracer which collects // 4 byte-identifiers of a tx, and implements vm.EVMLogger. -func newFourByteTracer(ctx *tracers.Context) tracers.Tracer { +func newFourByteTracer(ctx *tracers.Context, _ json.RawMessage) (tracers.Tracer, error) { t := &fourByteTracer{ ids: make(map[string]int), } - return t + return t, nil } // isPrecompiled returns whether the addr is a precompile. Logic borrowed from newJsTracer in eth/tracers/js/tracer.go diff --git a/eth/tracers/native/call.go b/eth/tracers/native/call.go index d334e328a5ffb..7af0e658a8bf8 100644 --- a/eth/tracers/native/call.go +++ b/eth/tracers/native/call.go @@ -50,16 +50,27 @@ type callFrame struct { type callTracer struct { env *vm.EVM callstack []callFrame + config callTracerConfig interrupt uint32 // Atomic flag to signal execution interruption reason error // Textual reason for the interruption } +type callTracerConfig struct { + OnlyTopCall bool `json:"onlyTopCall"` // If true, call tracer won't collect any subcalls +} + // newCallTracer returns a native go tracer which tracks // call frames of a tx, and implements vm.EVMLogger. -func newCallTracer(ctx *tracers.Context) tracers.Tracer { +func newCallTracer(ctx *tracers.Context, cfg json.RawMessage) (tracers.Tracer, error) { + var config callTracerConfig + if cfg != nil { + if err := json.Unmarshal(cfg, &config); err != nil { + return nil, err + } + } // First callframe contains tx context info // and is populated on start and end. - return &callTracer{callstack: make([]callFrame, 1)} + return &callTracer{callstack: make([]callFrame, 1), config: config}, nil } // CaptureStart implements the EVMLogger interface to initialize the tracing operation. @@ -101,6 +112,9 @@ func (t *callTracer) CaptureFault(pc uint64, op vm.OpCode, gas, cost uint64, _ * // CaptureEnter is called when EVM enters a new scope (via call, create or selfdestruct). func (t *callTracer) CaptureEnter(typ vm.OpCode, from common.Address, to common.Address, input []byte, gas uint64, value *big.Int) { + if t.config.OnlyTopCall { + return + } // Skip if tracing was interrupted if atomic.LoadUint32(&t.interrupt) > 0 { t.env.Cancel() @@ -121,6 +135,9 @@ func (t *callTracer) CaptureEnter(typ vm.OpCode, from common.Address, to common. // CaptureExit is called when EVM exits a scope, even if the scope didn't // execute any code. func (t *callTracer) CaptureExit(output []byte, gasUsed uint64, err error) { + if t.config.OnlyTopCall { + return + } size := len(t.callstack) if size <= 1 { return diff --git a/eth/tracers/native/noop.go b/eth/tracers/native/noop.go index 0849fd74e9873..c252b2408fc9b 100644 --- a/eth/tracers/native/noop.go +++ b/eth/tracers/native/noop.go @@ -35,8 +35,8 @@ func init() { type noopTracer struct{} // newNoopTracer returns a new noop tracer. -func newNoopTracer(ctx *tracers.Context) tracers.Tracer { - return &noopTracer{} +func newNoopTracer(ctx *tracers.Context, _ json.RawMessage) (tracers.Tracer, error) { + return &noopTracer{}, nil } // CaptureStart implements the EVMLogger interface to initialize the tracing operation. diff --git a/eth/tracers/native/prestate.go b/eth/tracers/native/prestate.go index 4d289ca622107..b513f383b9c28 100644 --- a/eth/tracers/native/prestate.go +++ b/eth/tracers/native/prestate.go @@ -51,10 +51,10 @@ type prestateTracer struct { reason error // Textual reason for the interruption } -func newPrestateTracer(ctx *tracers.Context) tracers.Tracer { +func newPrestateTracer(ctx *tracers.Context, _ json.RawMessage) (tracers.Tracer, error) { // First callframe contains tx context info // and is populated on start and end. - return &prestateTracer{prestate: prestate{}} + return &prestateTracer{prestate: prestate{}}, nil } // CaptureStart implements the EVMLogger interface to initialize the tracing operation. diff --git a/eth/tracers/native/revertreason.go b/eth/tracers/native/revertreason.go new file mode 100644 index 0000000000000..d09b861009017 --- /dev/null +++ b/eth/tracers/native/revertreason.go @@ -0,0 +1,108 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package native + +import ( + "bytes" + "encoding/json" + "math/big" + "sync/atomic" + "time" + + "github.com/ethereum/go-ethereum/accounts/abi" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/eth/tracers" +) + +func init() { + register("revertReasonTracer", newRevertReasonTracer) +} + +var revertSelector = crypto.Keccak256([]byte("Error(string)"))[:4] + +// revertReasonTracer is a go implementation of the Tracer interface which +// track the error message or revert reason return by the contract. +type revertReasonTracer struct { + env *vm.EVM + revertReason string // The revert reason return from the tx, if tx success, empty string return + interrupt uint32 // Atomic flag to signal execution interruption + reason error // Textual reason for the interruption +} + +// newRevertReasonTracer returns a new revert reason tracer. +func newRevertReasonTracer(_ *tracers.Context, _ json.RawMessage) (tracers.Tracer, error) { + return &revertReasonTracer{}, nil +} + +// CaptureStart implements the EVMLogger interface to initialize the tracing operation. +func (t *revertReasonTracer) CaptureStart(env *vm.EVM, _ common.Address, _ common.Address, _ bool, _ []byte, _ uint64, _ *big.Int) { + t.env = env +} + +// CaptureEnd is called after the call finishes to finalize the tracing. +func (t *revertReasonTracer) CaptureEnd(output []byte, _ uint64, _ time.Duration, err error) { + if err != nil { + if err == vm.ErrExecutionReverted && len(output) > 4 && bytes.Equal(output[:4], revertSelector) { + errMsg, _ := abi.UnpackRevert(output) + t.revertReason = err.Error() + ": " + errMsg + } else { + t.revertReason = err.Error() + } + } +} + +// CaptureState implements the EVMLogger interface to trace a single step of VM execution. +func (t *revertReasonTracer) CaptureState(_ uint64, _ vm.OpCode, _, _ uint64, _ *vm.ScopeContext, _ []byte, _ int, _ error) { +} + +// CaptureFault implements the EVMLogger interface to trace an execution fault. +func (t *revertReasonTracer) CaptureFault(_ uint64, _ vm.OpCode, _, _ uint64, _ *vm.ScopeContext, _ int, _ error) { +} + +// CaptureEnter is called when EVM enters a new scope (via call, create or selfdestruct). +func (t *revertReasonTracer) CaptureEnter(_ vm.OpCode, _ common.Address, _ common.Address, _ []byte, _ uint64, _ *big.Int) { + // Skip if tracing was interrupted + if atomic.LoadUint32(&t.interrupt) > 0 { + t.env.Cancel() + return + } +} + +// CaptureExit is called when EVM exits a scope, even if the scope didn't +// execute any code. +func (t *revertReasonTracer) CaptureExit(_ []byte, _ uint64, _ error) {} + +func (t *revertReasonTracer) CaptureTxStart(_ uint64) {} + +func (t *revertReasonTracer) CaptureTxEnd(_ uint64) {} + +// GetResult returns an error message json object. +func (t *revertReasonTracer) GetResult() (json.RawMessage, error) { + res, err := json.Marshal(t.revertReason) + if err != nil { + return nil, err + } + return res, t.reason +} + +// Stop terminates execution of the tracer at the first opportune moment. +func (t *revertReasonTracer) Stop(err error) { + t.reason = err + atomic.StoreUint32(&t.interrupt, 1) +} diff --git a/eth/tracers/native/tracer.go b/eth/tracers/native/tracer.go index 3bab870ea510c..9587caf19fd1f 100644 --- a/eth/tracers/native/tracer.go +++ b/eth/tracers/native/tracer.go @@ -35,6 +35,7 @@ func init() { package native import ( + "encoding/json" "errors" "github.com/ethereum/go-ethereum/eth/tracers" @@ -46,7 +47,7 @@ func init() { } // ctorFn is the constructor signature of a native tracer. -type ctorFn = func(*tracers.Context) tracers.Tracer +type ctorFn = func(*tracers.Context, json.RawMessage) (tracers.Tracer, error) /* ctors is a map of package-local tracer constructors. @@ -71,12 +72,12 @@ func register(name string, ctor ctorFn) { } // lookup returns a tracer, if one can be matched to the given name. -func lookup(name string, ctx *tracers.Context) (tracers.Tracer, error) { +func lookup(name string, ctx *tracers.Context, cfg json.RawMessage) (tracers.Tracer, error) { if ctors == nil { ctors = make(map[string]ctorFn) } if ctor, ok := ctors[name]; ok { - return ctor(ctx), nil + return ctor(ctx, cfg) } return nil, errors.New("no tracer found") } diff --git a/eth/tracers/tracers.go b/eth/tracers/tracers.go index e7073e7d2edf5..3d2d1256c0914 100644 --- a/eth/tracers/tracers.go +++ b/eth/tracers/tracers.go @@ -42,7 +42,7 @@ type Tracer interface { Stop(err error) } -type lookupFunc func(string, *Context) (Tracer, error) +type lookupFunc func(string, *Context, json.RawMessage) (Tracer, error) var ( lookups []lookupFunc @@ -62,9 +62,9 @@ func RegisterLookup(wildcard bool, lookup lookupFunc) { // New returns a new instance of a tracer, by iterating through the // registered lookups. -func New(code string, ctx *Context) (Tracer, error) { +func New(code string, ctx *Context, cfg json.RawMessage) (Tracer, error) { for _, lookup := range lookups { - if tracer, err := lookup(code, ctx); err == nil { + if tracer, err := lookup(code, ctx, cfg); err == nil { return tracer, nil } } diff --git a/eth/tracers/tracers_test.go b/eth/tracers/tracers_test.go index ce9289dd756b2..12e01abae4034 100644 --- a/eth/tracers/tracers_test.go +++ b/eth/tracers/tracers_test.go @@ -21,7 +21,6 @@ import ( "testing" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" @@ -32,20 +31,6 @@ import ( "github.com/ethereum/go-ethereum/tests" ) -// callTrace is the result of a callTracer run. -type callTrace struct { - Type string `json:"type"` - From common.Address `json:"from"` - To common.Address `json:"to"` - Input hexutil.Bytes `json:"input"` - Output hexutil.Bytes `json:"output"` - Gas *hexutil.Uint64 `json:"gas,omitempty"` - GasUsed *hexutil.Uint64 `json:"gasUsed,omitempty"` - Value *hexutil.Big `json:"value,omitempty"` - Error string `json:"error,omitempty"` - Calls []callTrace `json:"calls,omitempty"` -} - func BenchmarkTransactionTrace(b *testing.B) { key, _ := crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") from := crypto.PubkeyToAddress(key.PublicKey) diff --git a/ethclient/ethclient.go b/ethclient/ethclient.go index 24edd8648ef39..8a001843187bb 100644 --- a/ethclient/ethclient.go +++ b/ethclient/ethclient.go @@ -505,6 +505,38 @@ func (ec *Client) SuggestGasTipCap(ctx context.Context) (*big.Int, error) { return (*big.Int)(&hex), nil } +type feeHistoryResultMarshaling struct { + OldestBlock *hexutil.Big `json:"oldestBlock"` + Reward [][]*hexutil.Big `json:"reward,omitempty"` + BaseFee []*hexutil.Big `json:"baseFeePerGas,omitempty"` + GasUsedRatio []float64 `json:"gasUsedRatio"` +} + +// FeeHistory retrieves the fee market history. +func (ec *Client) FeeHistory(ctx context.Context, blockCount uint64, lastBlock *big.Int, rewardPercentiles []float64) (*ethereum.FeeHistory, error) { + var res feeHistoryResultMarshaling + if err := ec.c.CallContext(ctx, &res, "eth_feeHistory", hexutil.Uint(blockCount), toBlockNumArg(lastBlock), rewardPercentiles); err != nil { + return nil, err + } + reward := make([][]*big.Int, len(res.Reward)) + for i, r := range res.Reward { + reward[i] = make([]*big.Int, len(r)) + for j, r := range r { + reward[i][j] = (*big.Int)(r) + } + } + baseFee := make([]*big.Int, len(res.BaseFee)) + for i, b := range res.BaseFee { + baseFee[i] = (*big.Int)(b) + } + return ðereum.FeeHistory{ + OldestBlock: (*big.Int)(res.OldestBlock), + Reward: reward, + BaseFee: baseFee, + GasUsedRatio: res.GasUsedRatio, + }, nil +} + // EstimateGas tries to estimate the gas needed to execute a specific transaction based on // the current pending state of the backend blockchain. There is no guarantee that this is // the true gas limit requirement as other transactions may be added or removed by miners, diff --git a/ethclient/ethclient_test.go b/ethclient/ethclient_test.go index 4a8727b37478e..f2f4a5765d133 100644 --- a/ethclient/ethclient_test.go +++ b/ethclient/ethclient_test.go @@ -248,7 +248,7 @@ func generateTestChain() []*types.Block { g.AddTx(testTx2) } } - gblock := genesis.ToBlock(db) + gblock := genesis.MustCommit(db) engine := ethash.NewFaker() blocks, _ := core.GenerateChain(genesis.Config, gblock, engine, db, 2, generate) blocks = append([]*types.Block{gblock}, blocks...) @@ -508,6 +508,29 @@ func testStatusFunctions(t *testing.T, client *rpc.Client) { if gasTipCap.Cmp(big.NewInt(234375000)) != 0 { t.Fatalf("unexpected gas tip cap: %v", gasTipCap) } + + // FeeHistory + history, err := ec.FeeHistory(context.Background(), 1, big.NewInt(2), []float64{95, 99}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + want := ðereum.FeeHistory{ + OldestBlock: big.NewInt(2), + Reward: [][]*big.Int{ + { + big.NewInt(234375000), + big.NewInt(234375000), + }, + }, + BaseFee: []*big.Int{ + big.NewInt(765625000), + big.NewInt(671627818), + }, + GasUsedRatio: []float64{0.008912678667376286}, + } + if !reflect.DeepEqual(history, want) { + t.Fatalf("FeeHistory result doesn't match expected: (got: %v, want: %v)", history, want) + } } func testCallContractAtHash(t *testing.T, client *rpc.Client) { @@ -558,7 +581,7 @@ func testCallContract(t *testing.T, client *rpc.Client) { if _, err := ec.CallContract(context.Background(), msg, big.NewInt(1)); err != nil { t.Fatalf("unexpected error: %v", err) } - // PendingCallCOntract + // PendingCallContract if _, err := ec.PendingCallContract(context.Background(), msg); err != nil { t.Fatalf("unexpected error: %v", err) } diff --git a/ethclient/gethclient/gethclient.go b/ethclient/gethclient/gethclient.go index 7af2bf45d791e..a86f4339f425c 100644 --- a/ethclient/gethclient/gethclient.go +++ b/ethclient/gethclient/gethclient.go @@ -79,7 +79,6 @@ type StorageResult struct { // GetProof returns the account and storage values of the specified account including the Merkle-proof. // The block number can be nil, in which case the value is taken from the latest known block. func (ec *Client) GetProof(ctx context.Context, account common.Address, keys []string, blockNumber *big.Int) (*AccountResult, error) { - type storageResult struct { Key string `json:"key"` Value *hexutil.Big `json:"value"` diff --git a/ethclient/gethclient/gethclient_test.go b/ethclient/gethclient/gethclient_test.go index 758acc085b377..a0f4eaaf5db4c 100644 --- a/ethclient/gethclient/gethclient_test.go +++ b/ethclient/gethclient/gethclient_test.go @@ -31,6 +31,7 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/eth/ethconfig" + "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/params" @@ -60,6 +61,12 @@ func newTestBackend(t *testing.T) (*node.Node, []*types.Block) { if err != nil { t.Fatalf("can't create new ethereum service: %v", err) } + filterSystem := filters.NewFilterSystem(ethservice.APIBackend, filters.Config{}) + n.RegisterAPIs([]rpc.API{{ + Namespace: "eth", + Service: filters.NewFilterAPI(filterSystem, false), + }}) + // Import the test chain. if err := n.Start(); err != nil { t.Fatalf("can't start test node: %v", err) @@ -83,7 +90,7 @@ func generateTestChain() (*core.Genesis, []*types.Block) { g.OffsetTime(5) g.SetExtra([]byte("test")) } - gblock := genesis.ToBlock(db) + gblock := genesis.MustCommit(db) engine := ethash.NewFaker() blocks, _ := core.GenerateChain(config, gblock, engine, db, 1, generate) blocks = append([]*types.Block{gblock}, blocks...) @@ -222,7 +229,6 @@ func testGetProof(t *testing.T, client *rpc.Client) { if proof.Key != testSlot.String() { t.Fatalf("invalid storage proof key, want: %v, got: %v", testSlot.String(), proof.Key) } - } func testGCStats(t *testing.T, client *rpc.Client) { diff --git a/ethdb/database.go b/ethdb/database.go index e8faa2d868cc8..361218f247421 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -142,7 +142,9 @@ type AncientWriteOp interface { // AncientStater wraps the Stat method of a backing data store. type AncientStater interface { - // AncientDatadir returns the root directory path of the ancient store. + // AncientDatadir returns the path of root ancient directory. Empty string + // will be returned if ancient store is not enabled at all. The returned + // path can be used to construct the path of other freezers. AncientDatadir() (string, error) } @@ -172,7 +174,6 @@ type Stater interface { type AncientStore interface { AncientReader AncientWriter - AncientStater io.Closer } diff --git a/ethdb/memorydb/memorydb.go b/ethdb/memorydb/memorydb.go index e94570cb3f0ec..7e4fd7e5e7f0e 100644 --- a/ethdb/memorydb/memorydb.go +++ b/ethdb/memorydb/memorydb.go @@ -66,7 +66,7 @@ func NewWithCap(size int) *Database { } // Close deallocates the internal map and ensures any consecutive data access op -// failes with an error. +// fails with an error. func (db *Database) Close() error { db.lock.Lock() defer db.lock.Unlock() diff --git a/ethstats/ethstats_test.go b/ethstats/ethstats_test.go index 0692ecdae9bec..60322f765439e 100644 --- a/ethstats/ethstats_test.go +++ b/ethstats/ethstats_test.go @@ -79,5 +79,4 @@ func TestParseEthstatsURL(t *testing.T) { t.Errorf("case=%d mismatch host value, got: %v ,want: %v", i, host, c.host) } } - } diff --git a/go.mod b/go.mod index 46dce52a101e6..db3a5148e37fb 100644 --- a/go.mod +++ b/go.mod @@ -1,10 +1,9 @@ module github.com/0xTomoyo/go-ethereum -go 1.16 +go 1.17 require ( github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0 - github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 // indirect github.com/VictoriaMetrics/fastcache v1.6.0 github.com/aws/aws-sdk-go-v2 v1.2.0 github.com/aws/aws-sdk-go-v2/config v1.1.1 @@ -16,7 +15,6 @@ require ( github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f github.com/davecgh/go-spew v1.1.1 github.com/deckarep/golang-set v1.8.0 - github.com/deepmap/oapi-codegen v1.8.2 // indirect github.com/docker/docker v1.6.2 github.com/dop251/goja v0.0.0-20220405120441-9037c2b61cbf github.com/edsrzf/mmap-go v1.0.0 @@ -24,10 +22,9 @@ require ( github.com/fjl/gencodec v0.0.0-20220412091415-8bb9e558978c github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff - github.com/go-ole/go-ole v1.2.1 // indirect github.com/go-stack/stack v1.8.0 github.com/golang-jwt/jwt/v4 v4.3.0 - github.com/golang/protobuf v1.4.3 + github.com/golang/protobuf v1.5.2 github.com/golang/snappy v0.0.4 github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa github.com/google/uuid v1.2.0 @@ -40,15 +37,12 @@ require ( github.com/huin/goupnp v1.0.3 github.com/influxdata/influxdb v1.8.3 github.com/influxdata/influxdb-client-go/v2 v2.4.0 - github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097 // indirect github.com/jackpal/go-nat-pmp v1.0.2 github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e github.com/julienschmidt/httprouter v1.2.0 github.com/karalabe/usb v0.0.2 - github.com/kylelemons/godebug v1.1.0 // indirect github.com/mattn/go-colorable v0.1.8 github.com/mattn/go-isatty v0.0.12 - github.com/naoina/go-stringutil v0.1.0 // indirect github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416 github.com/olekukonko/tablewriter v0.0.5 github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 @@ -57,17 +51,58 @@ require ( github.com/rs/cors v1.7.0 github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 - github.com/stretchr/testify v1.7.0 + github.com/stretchr/testify v1.7.2 + github.com/supranational/blst v0.3.8-0.20220526154634-513d2456b344 github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 - github.com/tklauser/go-sysconf v0.3.5 // indirect github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef + github.com/urfave/cli/v2 v2.10.2 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c - golang.org/x/sys v0.0.0-20211019181941-9d821ace8654 + golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a + golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 golang.org/x/text v0.3.7 golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023 gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce - gopkg.in/urfave/cli.v1 v1.20.0 +) + +require ( + github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.3 // indirect + github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.0.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.0.2 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.1.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.1.1 // indirect + github.com/aws/smithy-go v1.1.0 // indirect + github.com/cespare/xxhash/v2 v2.1.1 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect + github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect + github.com/deepmap/oapi-codegen v1.8.2 // indirect + github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91 // indirect + github.com/garslo/gogen v0.0.0-20170306192744-1d203ffc1f61 // indirect + github.com/go-logfmt/logfmt v0.4.0 // indirect + github.com/go-ole/go-ole v1.2.1 // indirect + github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect + github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097 // indirect + github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect + github.com/mattn/go-runewidth v0.0.9 // indirect + github.com/mitchellh/mapstructure v1.4.1 // indirect + github.com/mitchellh/pointerstructure v1.2.0 // indirect + github.com/naoina/go-stringutil v0.1.0 // indirect + github.com/opentracing/opentracing-go v1.1.0 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/tklauser/go-sysconf v0.3.5 // indirect + github.com/tklauser/numcpus v0.2.2 // indirect + github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect + golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57 // indirect + golang.org/x/net v0.0.0-20220607020251-c690dde0001d // indirect + golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df // indirect + google.golang.org/protobuf v1.26.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 0d793209b0e56..4b27867fbc792 100644 --- a/go.sum +++ b/go.sum @@ -25,6 +25,7 @@ github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.3/go.mod h1:KLF4gFr6DcKFZwSu github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0 h1:Px2UA+2RvSSvv+RvJNuUB6n7rs5Wsel4dXLe90Um2n4= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0/go.mod h1:tPaiy8S5bQ+S5sOiDlINkp7+Ef339+Nz5L5XO+cnOHo= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v1.1.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= @@ -83,6 +84,8 @@ github.com/consensys/bavard v0.1.8-0.20210406032232-f3452dc9b572/go.mod h1:Bpd0/ github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f h1:C43yEtQ6NIf4ftFXD/V55gnGFgPbMQobd//YlnLjUJ8= github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f/go.mod h1:815PAHg3wvysy0SyIqanF8gZ0Y1wjk/hrDHD/iT88+Q= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= @@ -124,8 +127,9 @@ github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 h1:FtmdgXiUlNeRsoNMFlK github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= +github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/garslo/gogen v0.0.0-20170306192744-1d203ffc1f61 h1:IZqZOB2fydHte3kUgxrzK5E1fW7RQGeDwE8F/ZZnUYc= github.com/garslo/gogen v0.0.0-20170306192744-1d203ffc1f61/go.mod h1:Q0X6pkwTILDlzrGEckF6HKjXe48EgsY/l7K7vhY4MW8= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff h1:tY80oXqGNY4FhTFhk+o9oFHGINQ/+vhlm8HFzi6znCI= @@ -152,6 +156,7 @@ github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5Nq github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= @@ -174,8 +179,9 @@ github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrU github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= @@ -189,14 +195,16 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa h1:Q75Upo5UN4JbPFURXZ8nLKYUvF85dyFRop/vQ0Rv+64= github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -223,6 +231,7 @@ github.com/huin/goupnp v1.0.3 h1:N8No57ls+MnjlB+JPiCVSOyy/ot7MJTqlo7rn+NYSqQ= github.com/huin/goupnp v1.0.3/go.mod h1:ZxNlw5WqJj6wSsRK5+YfflQGXYfccj5VgQsMNixHM7Y= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/flux v0.65.1/go.mod h1:J754/zds0vvpfwuq7Gc2wRdVwEodfpCFM7mYlOw2LqY= github.com/influxdata/influxdb v1.8.3 h1:WEypI1BQFTT4teLM+1qkEcvUi0dAvopAI/ir0vAiBg8= @@ -311,19 +320,29 @@ github.com/naoina/go-stringutil v0.1.0 h1:rCUeRUHjBjGTSHl0VC00jUPLz8/F9dDzYI70Hz github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0= github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416 h1:shk/vn9oCoOTmwcouEdwIeOtOGA/ELRUw/GwvxwfT+0= github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= -github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.3 h1:OoxbjfXVZyod1fmWYhI7SEyaD8B00ynP3T+D5GiyHOY= github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.1 h1:K0jcRCwNQM3vFGh1ppMtDh/+7ApJrjldlX8fA0jDTLQ= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw= +github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.0.3-0.20180606204148-bd9c31933947/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= @@ -360,6 +379,8 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/segmentio/kafka-go v0.1.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= github.com/segmentio/kafka-go v0.2.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= @@ -382,10 +403,19 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.2 h1:4jaiDzPyXQvSd7D0EjG45355tLlV3VOECpq10pLC+8s= +github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= +github.com/supranational/blst v0.3.8-0.20220526154634-513d2456b344 h1:m+8fKfQwCAy1QjzINvKe/pYtLjo2dl59x2w9YSEJxuY= +github.com/supranational/blst v0.3.8-0.20220526154634-513d2456b344/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= +github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= +github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= +github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a h1:1ur3QoCqvE5fl+nylMaIr9PVV1w343YRDtsy+Rwu7XI= +github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a/go.mod h1:RRCYJbIwD5jmqPI9XoAFR0OcDxqUctll6zUj/+B4S48= +github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d h1:vfofYNRScrDdvS342BElfbETmL1Aiz3i2t0zfRj16Hs= +github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d/go.mod h1:RRCYJbIwD5jmqPI9XoAFR0OcDxqUctll6zUj/+B4S48= github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tklauser/go-sysconf v0.3.5 h1:uu3Xl4nkLzQfXNsWn15rPc/HQCJKObbt1dKJeWp3vU4= github.com/tklauser/go-sysconf v0.3.5/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI= @@ -394,11 +424,15 @@ github.com/tklauser/numcpus v0.2.2/go.mod h1:x3qojaO3uyYt0i56EW/VUYs7uBvdl2fkfZF github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef h1:wHSqTBrZW24CsNJDfeh9Ex6Pm0Rcpc7qrgKBiL44vF4= github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= +github.com/urfave/cli/v2 v2.10.2 h1:x3p8awjp/2arX+Nl/G2040AZpOCHS/eMJJ1/a+mye4Y= +github.com/urfave/cli/v2 v2.10.2/go.mod h1:f8iq5LtQ/bLxafbdBSLPPNsgaW0l/2fYYEHhAyPlwvo= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= +github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= +github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= @@ -475,10 +509,13 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210220033124-5f55cee0dc0d/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f h1:OfiFi4JbukWwe3lzw+xunroH1mnC1e2Gy5cxNJApiSY= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220607020251-c690dde0001d h1:4SFsTMi4UahlKoloni7L4eYzhFRifURQLw+yv0QDCx8= +golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -523,6 +560,7 @@ golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200826173525-f9321e4c35a6/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -530,11 +568,15 @@ golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210420205809-ac73e9fd8988/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211019181941-9d821ace8654 h1:id054HUawV2/6IGm2IV8KZQjqtwAOo2CYlOToYqa0d0= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -576,14 +618,16 @@ golang.org/x/tools v0.0.0-20191126055441-b0650ceb63d9/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200108203644-89082a384178/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023 h1:0c3L82FDQ5rt1bjTBlchS8t6RQ6299/+5bWMnRLh+uI= golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df h1:5Pf6pFKu98ODmgnpvkJ3kFUOQGGLIzLIkbzUHp47618= +golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.6.0/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU= @@ -626,8 +670,10 @@ google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -639,8 +685,6 @@ gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7 gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/urfave/cli.v1 v1.20.0 h1:NdAVW6RYxDif9DhDHaAortIu956m2c0v+09AZBPTbE0= -gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -650,8 +694,9 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/graphql/graphql.go b/graphql/graphql.go index 0654fd1af3885..356ff669fb16d 100644 --- a/graphql/graphql.go +++ b/graphql/graphql.go @@ -22,6 +22,7 @@ import ( "errors" "fmt" "math/big" + "sort" "strconv" "github.com/ethereum/go-ethereum" @@ -76,14 +77,14 @@ func (b *Long) UnmarshalGraphQL(input interface{}) error { // Account represents an Ethereum account at a particular block. type Account struct { - backend ethapi.Backend + r *Resolver address common.Address blockNrOrHash rpc.BlockNumberOrHash } // getState fetches the StateDB object for an account. func (a *Account) getState(ctx context.Context) (*state.StateDB, error) { - state, _, err := a.backend.StateAndHeaderByNumberOrHash(ctx, a.blockNrOrHash) + state, _, err := a.r.backend.StateAndHeaderByNumberOrHash(ctx, a.blockNrOrHash) return state, err } @@ -106,7 +107,7 @@ func (a *Account) Balance(ctx context.Context) (hexutil.Big, error) { func (a *Account) TransactionCount(ctx context.Context) (hexutil.Uint64, error) { // Ask transaction pool for the nonce which includes pending transactions if blockNr, ok := a.blockNrOrHash.Number(); ok && blockNr == rpc.PendingBlockNumber { - nonce, err := a.backend.GetPoolNonce(ctx, a.address) + nonce, err := a.r.backend.GetPoolNonce(ctx, a.address) if err != nil { return 0, err } @@ -137,7 +138,7 @@ func (a *Account) Storage(ctx context.Context, args struct{ Slot common.Hash }) // Log represents an individual log message. All arguments are mandatory. type Log struct { - backend ethapi.Backend + r *Resolver transaction *Transaction log *types.Log } @@ -148,7 +149,7 @@ func (l *Log) Transaction(ctx context.Context) *Transaction { func (l *Log) Account(ctx context.Context, args BlockNumberArgs) *Account { return &Account{ - backend: l.backend, + r: l.r, address: l.log.Address, blockNrOrHash: args.NumberOrLatest(), } @@ -183,30 +184,30 @@ func (at *AccessTuple) StorageKeys(ctx context.Context) []common.Hash { // Transaction represents an Ethereum transaction. // backend and hash are mandatory; all others will be fetched when required. type Transaction struct { - backend ethapi.Backend - hash common.Hash - tx *types.Transaction - block *Block - index uint64 + r *Resolver + hash common.Hash + tx *types.Transaction + block *Block + index uint64 } // resolve returns the internal transaction object, fetching it if needed. func (t *Transaction) resolve(ctx context.Context) (*types.Transaction, error) { if t.tx == nil { // Try to return an already finalized transaction - tx, blockHash, _, index, err := t.backend.GetTransaction(ctx, t.hash) + tx, blockHash, _, index, err := t.r.backend.GetTransaction(ctx, t.hash) if err == nil && tx != nil { t.tx = tx blockNrOrHash := rpc.BlockNumberOrHashWithHash(blockHash, false) t.block = &Block{ - backend: t.backend, + r: t.r, numberOrHash: &blockNrOrHash, } t.index = index return t.tx, nil } // No finalized transaction, try to retrieve it from the pool - t.tx = t.backend.GetPoolTransaction(t.hash) + t.tx = t.r.backend.GetPoolTransaction(t.hash) } return t.tx, nil } @@ -354,7 +355,7 @@ func (t *Transaction) To(ctx context.Context, args BlockNumberArgs) (*Account, e return nil, nil } return &Account{ - backend: t.backend, + r: t.r, address: *to, blockNrOrHash: args.NumberOrLatest(), }, nil @@ -365,10 +366,10 @@ func (t *Transaction) From(ctx context.Context, args BlockNumberArgs) (*Account, if err != nil || tx == nil { return nil, err } - signer := types.LatestSigner(t.backend.ChainConfig()) + signer := types.LatestSigner(t.r.backend.ChainConfig()) from, _ := types.Sender(signer, tx) return &Account{ - backend: t.backend, + r: t.r, address: from, blockNrOrHash: args.NumberOrLatest(), }, nil @@ -443,24 +444,51 @@ func (t *Transaction) CreatedContract(ctx context.Context, args BlockNumberArgs) return nil, err } return &Account{ - backend: t.backend, + r: t.r, address: receipt.ContractAddress, blockNrOrHash: args.NumberOrLatest(), }, nil } func (t *Transaction) Logs(ctx context.Context) (*[]*Log, error) { - receipt, err := t.getReceipt(ctx) - if err != nil || receipt == nil { + if _, err := t.resolve(ctx); err != nil { + return nil, err + } + if t.block == nil { + return nil, nil + } + if _, ok := t.block.numberOrHash.Hash(); !ok { + header, err := t.r.backend.HeaderByNumberOrHash(ctx, *t.block.numberOrHash) + if err != nil { + return nil, err + } + hash := header.Hash() + t.block.numberOrHash.BlockHash = &hash + } + return t.getLogs(ctx) +} + +// getLogs returns log objects for the given tx. +// Assumes block hash is resolved. +func (t *Transaction) getLogs(ctx context.Context) (*[]*Log, error) { + var ( + hash, _ = t.block.numberOrHash.Hash() + filter = t.r.filterSystem.NewBlockFilter(hash, nil, nil) + logs, err = filter.Logs(ctx) + ) + if err != nil { return nil, err } - ret := make([]*Log, 0, len(receipt.Logs)) - for _, log := range receipt.Logs { + var ret []*Log + // Select tx logs from all block logs + ix := sort.Search(len(logs), func(i int) bool { return uint64(logs[i].TxIndex) >= t.index }) + for ix < len(logs) && uint64(logs[ix].TxIndex) == t.index { ret = append(ret, &Log{ - backend: t.backend, + r: t.r, transaction: t, - log: log, + log: logs[ix], }) + ix++ } return &ret, nil } @@ -539,7 +567,7 @@ type BlockType int // backend, and numberOrHash are mandatory. All other fields are lazily fetched // when required. type Block struct { - backend ethapi.Backend + r *Resolver numberOrHash *rpc.BlockNumberOrHash hash common.Hash header *types.Header @@ -558,7 +586,7 @@ func (b *Block) resolve(ctx context.Context) (*types.Block, error) { b.numberOrHash = &latest } var err error - b.block, err = b.backend.BlockByNumberOrHash(ctx, *b.numberOrHash) + b.block, err = b.r.backend.BlockByNumberOrHash(ctx, *b.numberOrHash) if b.block != nil && b.header == nil { b.header = b.block.Header() if hash, ok := b.numberOrHash.Hash(); ok { @@ -578,9 +606,9 @@ func (b *Block) resolveHeader(ctx context.Context) (*types.Header, error) { var err error if b.header == nil { if b.hash != (common.Hash{}) { - b.header, err = b.backend.HeaderByHash(ctx, b.hash) + b.header, err = b.r.backend.HeaderByHash(ctx, b.hash) } else { - b.header, err = b.backend.HeaderByNumberOrHash(ctx, *b.numberOrHash) + b.header, err = b.r.backend.HeaderByNumberOrHash(ctx, *b.numberOrHash) } } return b.header, err @@ -598,7 +626,7 @@ func (b *Block) resolveReceipts(ctx context.Context) ([]*types.Receipt, error) { } hash = header.Hash() } - receipts, err := b.backend.GetReceipts(ctx, hash) + receipts, err := b.r.backend.GetReceipts(ctx, hash) if err != nil { return nil, err } @@ -659,7 +687,7 @@ func (b *Block) NextBaseFeePerGas(ctx context.Context) (*hexutil.Big, error) { if err != nil { return nil, err } - chaincfg := b.backend.ChainConfig() + chaincfg := b.r.backend.ChainConfig() if header.BaseFee == nil { // Make sure next block doesn't enable EIP-1559 if !chaincfg.IsLondon(new(big.Int).Add(header.Number, common.Big1)) { @@ -679,7 +707,7 @@ func (b *Block) Parent(ctx context.Context) (*Block, error) { } num := rpc.BlockNumberOrHashWithNumber(rpc.BlockNumber(b.header.Number.Uint64() - 1)) return &Block{ - backend: b.backend, + r: b.r, numberOrHash: &num, hash: b.header.ParentHash, }, nil @@ -767,7 +795,7 @@ func (b *Block) Ommers(ctx context.Context) (*[]*Block, error) { for _, uncle := range block.Uncles() { blockNumberOrHash := rpc.BlockNumberOrHashWithHash(uncle.Hash(), false) ret = append(ret, &Block{ - backend: b.backend, + r: b.r, numberOrHash: &blockNumberOrHash, header: uncle, }) @@ -800,7 +828,7 @@ func (b *Block) TotalDifficulty(ctx context.Context) (hexutil.Big, error) { } h = header.Hash() } - td := b.backend.GetTd(ctx, h) + td := b.r.backend.GetTd(ctx, h) if td == nil { return hexutil.Big{}, fmt.Errorf("total difficulty not found %x", b.hash) } @@ -853,7 +881,7 @@ func (b *Block) Miner(ctx context.Context, args BlockNumberArgs) (*Account, erro return nil, err } return &Account{ - backend: b.backend, + r: b.r, address: header.Coinbase, blockNrOrHash: args.NumberOrLatest(), }, nil @@ -876,11 +904,11 @@ func (b *Block) Transactions(ctx context.Context) (*[]*Transaction, error) { ret := make([]*Transaction, 0, len(block.Transactions())) for i, tx := range block.Transactions() { ret = append(ret, &Transaction{ - backend: b.backend, - hash: tx.Hash(), - tx: tx, - block: b, - index: uint64(i), + r: b.r, + hash: tx.Hash(), + tx: tx, + block: b, + index: uint64(i), }) } return &ret, nil @@ -897,11 +925,11 @@ func (b *Block) TransactionAt(ctx context.Context, args struct{ Index int32 }) ( } tx := txs[args.Index] return &Transaction{ - backend: b.backend, - hash: tx.Hash(), - tx: tx, - block: b, - index: uint64(args.Index), + r: b.r, + hash: tx.Hash(), + tx: tx, + block: b, + index: uint64(args.Index), }, nil } @@ -917,7 +945,7 @@ func (b *Block) OmmerAt(ctx context.Context, args struct{ Index int32 }) (*Block uncle := uncles[args.Index] blockNumberOrHash := rpc.BlockNumberOrHashWithHash(uncle.Hash(), false) return &Block{ - backend: b.backend, + r: b.r, numberOrHash: &blockNumberOrHash, header: uncle, }, nil @@ -944,7 +972,7 @@ type BlockFilterCriteria struct { // runFilter accepts a filter and executes it, returning all its results as // `Log` objects. -func runFilter(ctx context.Context, be ethapi.Backend, filter *filters.Filter) ([]*Log, error) { +func runFilter(ctx context.Context, r *Resolver, filter *filters.Filter) ([]*Log, error) { logs, err := filter.Logs(ctx) if err != nil || logs == nil { return nil, err @@ -952,8 +980,8 @@ func runFilter(ctx context.Context, be ethapi.Backend, filter *filters.Filter) ( ret := make([]*Log, 0, len(logs)) for _, log := range logs { ret = append(ret, &Log{ - backend: be, - transaction: &Transaction{backend: be, hash: log.TxHash}, + r: r, + transaction: &Transaction{r: r, hash: log.TxHash}, log: log, }) } @@ -978,10 +1006,10 @@ func (b *Block) Logs(ctx context.Context, args struct{ Filter BlockFilterCriteri hash = header.Hash() } // Construct the range filter - filter := filters.NewBlockFilter(b.backend, hash, addresses, topics) + filter := b.r.filterSystem.NewBlockFilter(hash, addresses, topics) // Run the filter and return all the logs - return runFilter(ctx, b.backend, filter) + return runFilter(ctx, b.r, filter) } func (b *Block) Account(ctx context.Context, args struct { @@ -994,7 +1022,7 @@ func (b *Block) Account(ctx context.Context, args struct { } } return &Account{ - backend: b.backend, + r: b.r, address: args.Address, blockNrOrHash: *b.numberOrHash, }, nil @@ -1041,7 +1069,7 @@ func (b *Block) Call(ctx context.Context, args struct { return nil, err } } - result, err := ethapi.DoCall(ctx, b.backend, args.Data, *b.numberOrHash, nil, b.backend.RPCEVMTimeout(), b.backend.RPCGasCap()) + result, err := ethapi.DoCall(ctx, b.r.backend, args.Data, *b.numberOrHash, nil, b.r.backend.RPCEVMTimeout(), b.r.backend.RPCGasCap()) if err != nil { return nil, err } @@ -1066,31 +1094,31 @@ func (b *Block) EstimateGas(ctx context.Context, args struct { return 0, err } } - gas, err := ethapi.DoEstimateGas(ctx, b.backend, args.Data, *b.numberOrHash, b.backend.RPCGasCap()) + gas, err := ethapi.DoEstimateGas(ctx, b.r.backend, args.Data, *b.numberOrHash, b.r.backend.RPCGasCap()) return Long(gas), err } type Pending struct { - backend ethapi.Backend + r *Resolver } func (p *Pending) TransactionCount(ctx context.Context) (int32, error) { - txs, err := p.backend.GetPoolTransactions() + txs, err := p.r.backend.GetPoolTransactions() return int32(len(txs)), err } func (p *Pending) Transactions(ctx context.Context) (*[]*Transaction, error) { - txs, err := p.backend.GetPoolTransactions() + txs, err := p.r.backend.GetPoolTransactions() if err != nil { return nil, err } ret := make([]*Transaction, 0, len(txs)) for i, tx := range txs { ret = append(ret, &Transaction{ - backend: p.backend, - hash: tx.Hash(), - tx: tx, - index: uint64(i), + r: p.r, + hash: tx.Hash(), + tx: tx, + index: uint64(i), }) } return &ret, nil @@ -1101,7 +1129,7 @@ func (p *Pending) Account(ctx context.Context, args struct { }) *Account { pendingBlockNr := rpc.BlockNumberOrHashWithNumber(rpc.PendingBlockNumber) return &Account{ - backend: p.backend, + r: p.r, address: args.Address, blockNrOrHash: pendingBlockNr, } @@ -1111,7 +1139,7 @@ func (p *Pending) Call(ctx context.Context, args struct { Data ethapi.TransactionArgs }) (*CallResult, error) { pendingBlockNr := rpc.BlockNumberOrHashWithNumber(rpc.PendingBlockNumber) - result, err := ethapi.DoCall(ctx, p.backend, args.Data, pendingBlockNr, nil, p.backend.RPCEVMTimeout(), p.backend.RPCGasCap()) + result, err := ethapi.DoCall(ctx, p.r.backend, args.Data, pendingBlockNr, nil, p.r.backend.RPCEVMTimeout(), p.r.backend.RPCGasCap()) if err != nil { return nil, err } @@ -1131,13 +1159,14 @@ func (p *Pending) EstimateGas(ctx context.Context, args struct { Data ethapi.TransactionArgs }) (Long, error) { pendingBlockNr := rpc.BlockNumberOrHashWithNumber(rpc.PendingBlockNumber) - gas, err := ethapi.DoEstimateGas(ctx, p.backend, args.Data, pendingBlockNr, p.backend.RPCGasCap()) + gas, err := ethapi.DoEstimateGas(ctx, p.r.backend, args.Data, pendingBlockNr, p.r.backend.RPCGasCap()) return Long(gas), err } // Resolver is the top-level object in the GraphQL hierarchy. type Resolver struct { - backend ethapi.Backend + backend ethapi.Backend + filterSystem *filters.FilterSystem } func (r *Resolver) Block(ctx context.Context, args struct { @@ -1152,19 +1181,19 @@ func (r *Resolver) Block(ctx context.Context, args struct { number := rpc.BlockNumber(*args.Number) numberOrHash := rpc.BlockNumberOrHashWithNumber(number) block = &Block{ - backend: r.backend, + r: r, numberOrHash: &numberOrHash, } } else if args.Hash != nil { numberOrHash := rpc.BlockNumberOrHashWithHash(*args.Hash, false) block = &Block{ - backend: r.backend, + r: r, numberOrHash: &numberOrHash, } } else { numberOrHash := rpc.BlockNumberOrHashWithNumber(rpc.LatestBlockNumber) block = &Block{ - backend: r.backend, + r: r, numberOrHash: &numberOrHash, } } @@ -1199,7 +1228,7 @@ func (r *Resolver) Blocks(ctx context.Context, args struct { for i := from; i <= to; i++ { numberOrHash := rpc.BlockNumberOrHashWithNumber(i) block := &Block{ - backend: r.backend, + r: r, numberOrHash: &numberOrHash, } // Resolve the header to check for existence. @@ -1218,13 +1247,13 @@ func (r *Resolver) Blocks(ctx context.Context, args struct { } func (r *Resolver) Pending(ctx context.Context) *Pending { - return &Pending{r.backend} + return &Pending{r} } func (r *Resolver) Transaction(ctx context.Context, args struct{ Hash common.Hash }) (*Transaction, error) { tx := &Transaction{ - backend: r.backend, - hash: args.Hash, + r: r, + hash: args.Hash, } // Resolve the transaction; if it doesn't exist, return nil. t, err := tx.resolve(ctx) @@ -1284,8 +1313,8 @@ func (r *Resolver) Logs(ctx context.Context, args struct{ Filter FilterCriteria topics = *args.Filter.Topics } // Construct the range filter - filter := filters.NewRangeFilter(filters.Backend(r.backend), begin, end, addresses, topics) - return runFilter(ctx, r.backend, filter) + filter := r.filterSystem.NewRangeFilter(begin, end, addresses, topics) + return runFilter(ctx, r, filter) } func (r *Resolver) GasPrice(ctx context.Context) (hexutil.Big, error) { diff --git a/graphql/graphql_test.go b/graphql/graphql_test.go index 2768026b5197e..491c731521138 100644 --- a/graphql/graphql_test.go +++ b/graphql/graphql_test.go @@ -17,6 +17,8 @@ package graphql import ( + "context" + "encoding/json" "fmt" "io" "math/big" @@ -33,6 +35,7 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/eth/ethconfig" + "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/params" @@ -45,20 +48,26 @@ func TestBuildSchema(t *testing.T) { conf := node.DefaultConfig conf.DataDir = ddir stack, err := node.New(&conf) - defer stack.Close() if err != nil { t.Fatalf("could not create new node: %v", err) } + defer stack.Close() // Make sure the schema can be parsed and matched up to the object model. - if err := newHandler(stack, nil, []string{}, []string{}); err != nil { + if _, err := newHandler(stack, nil, nil, []string{}, []string{}); err != nil { t.Errorf("Could not construct GraphQL handler: %v", err) } } // Tests that a graphQL request is successfully handled when graphql is enabled on the specified endpoint func TestGraphQLBlockSerialization(t *testing.T) { - stack := createNode(t, true, false) + stack := createNode(t) defer stack.Close() + genesis := &core.Genesis{ + Config: params.AllEthashProtocolChanges, + GasLimit: 11500000, + Difficulty: big.NewInt(1048576), + } + newGQLService(t, stack, genesis, 10, func(i int, gen *core.BlockGen) {}) // start node if err := stack.Start(); err != nil { t.Fatalf("could not start node: %v", err) @@ -160,8 +169,55 @@ func TestGraphQLBlockSerialization(t *testing.T) { } func TestGraphQLBlockSerializationEIP2718(t *testing.T) { - stack := createNode(t, true, true) + // Account for signing txes + var ( + key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + address = crypto.PubkeyToAddress(key.PublicKey) + funds = big.NewInt(1000000000000000) + dad = common.HexToAddress("0x0000000000000000000000000000000000000dad") + ) + stack := createNode(t) defer stack.Close() + genesis := &core.Genesis{ + Config: params.AllEthashProtocolChanges, + GasLimit: 11500000, + Difficulty: big.NewInt(1048576), + Alloc: core.GenesisAlloc{ + address: {Balance: funds}, + // The address 0xdad sloads 0x00 and 0x01 + dad: { + Code: []byte{byte(vm.PC), byte(vm.PC), byte(vm.SLOAD), byte(vm.SLOAD)}, + Nonce: 0, + Balance: big.NewInt(0), + }, + }, + BaseFee: big.NewInt(params.InitialBaseFee), + } + signer := types.LatestSigner(genesis.Config) + newGQLService(t, stack, genesis, 1, func(i int, gen *core.BlockGen) { + gen.SetCoinbase(common.Address{1}) + tx, _ := types.SignNewTx(key, signer, &types.LegacyTx{ + Nonce: uint64(0), + To: &dad, + Value: big.NewInt(100), + Gas: 50000, + GasPrice: big.NewInt(params.InitialBaseFee), + }) + gen.AddTx(tx) + tx, _ = types.SignNewTx(key, signer, &types.AccessListTx{ + ChainID: genesis.Config.ChainID, + Nonce: uint64(1), + To: &dad, + Gas: 30000, + GasPrice: big.NewInt(params.InitialBaseFee), + Value: big.NewInt(50), + AccessList: types.AccessList{{ + Address: dad, + StorageKeys: []common.Hash{{0}}, + }}, + }) + gen.AddTx(tx) + }) // start node if err := stack.Start(); err != nil { t.Fatalf("could not start node: %v", err) @@ -197,7 +253,7 @@ func TestGraphQLBlockSerializationEIP2718(t *testing.T) { // Tests that a graphQL request is not handled successfully when graphql is not enabled on the specified endpoint func TestGraphQLHTTPOnSamePort_GQLRequest_Unsuccessful(t *testing.T) { - stack := createNode(t, false, false) + stack := createNode(t) defer stack.Close() if err := stack.Start(); err != nil { t.Fatalf("could not start node: %v", err) @@ -211,7 +267,59 @@ func TestGraphQLHTTPOnSamePort_GQLRequest_Unsuccessful(t *testing.T) { assert.Equal(t, http.StatusNotFound, resp.StatusCode) } -func createNode(t *testing.T, gqlEnabled bool, txEnabled bool) *node.Node { +func TestGraphQLTransactionLogs(t *testing.T) { + var ( + key, _ = crypto.GenerateKey() + addr = crypto.PubkeyToAddress(key.PublicKey) + dadStr = "0x0000000000000000000000000000000000000dad" + dad = common.HexToAddress(dadStr) + genesis = &core.Genesis{ + Config: params.AllEthashProtocolChanges, + GasLimit: 11500000, + Difficulty: big.NewInt(1048576), + Alloc: core.GenesisAlloc{ + addr: {Balance: big.NewInt(params.Ether)}, + dad: { + // LOG0(0, 0), LOG0(0, 0), RETURN(0, 0) + Code: common.Hex2Bytes("60006000a060006000a060006000f3"), + Nonce: 0, + Balance: big.NewInt(0), + }, + }, + } + signer = types.LatestSigner(genesis.Config) + stack = createNode(t) + ) + defer stack.Close() + + handler := newGQLService(t, stack, genesis, 1, func(i int, gen *core.BlockGen) { + tx, _ := types.SignNewTx(key, signer, &types.LegacyTx{To: &dad, Gas: 100000, GasPrice: big.NewInt(params.InitialBaseFee)}) + gen.AddTx(tx) + tx, _ = types.SignNewTx(key, signer, &types.LegacyTx{To: &dad, Nonce: 1, Gas: 100000, GasPrice: big.NewInt(params.InitialBaseFee)}) + gen.AddTx(tx) + tx, _ = types.SignNewTx(key, signer, &types.LegacyTx{To: &dad, Nonce: 2, Gas: 100000, GasPrice: big.NewInt(params.InitialBaseFee)}) + gen.AddTx(tx) + }) + // start node + if err := stack.Start(); err != nil { + t.Fatalf("could not start node: %v", err) + } + query := `{block { transactions { logs { account { address } } } } }` + res := handler.Schema.Exec(context.Background(), query, "", map[string]interface{}{}) + if res.Errors != nil { + t.Fatalf("graphql query failed: %v", res.Errors) + } + have, err := json.Marshal(res.Data) + if err != nil { + t.Fatalf("failed to encode graphql response: %s", err) + } + want := fmt.Sprintf(`{"block":{"transactions":[{"logs":[{"account":{"address":"%s"}},{"account":{"address":"%s"}}]},{"logs":[{"account":{"address":"%s"}},{"account":{"address":"%s"}}]},{"logs":[{"account":{"address":"%s"}},{"account":{"address":"%s"}}]}]}}`, dadStr, dadStr, dadStr, dadStr, dadStr, dadStr) + if string(have) != want { + t.Errorf("response unmatch. expected %s, got %s", want, have) + } +} + +func createNode(t *testing.T) *node.Node { stack, err := node.New(&node.Config{ HTTPHost: "127.0.0.1", HTTPPort: 0, @@ -221,25 +329,12 @@ func createNode(t *testing.T, gqlEnabled bool, txEnabled bool) *node.Node { if err != nil { t.Fatalf("could not create node: %v", err) } - if !gqlEnabled { - return stack - } - if !txEnabled { - createGQLService(t, stack) - } else { - createGQLServiceWithTransactions(t, stack) - } return stack } -func createGQLService(t *testing.T, stack *node.Node) { - // create backend +func newGQLService(t *testing.T, stack *node.Node, gspec *core.Genesis, genBlocks int, genfunc func(i int, gen *core.BlockGen)) *handler { ethConf := ðconfig.Config{ - Genesis: &core.Genesis{ - Config: params.AllEthashProtocolChanges, - GasLimit: 11500000, - Difficulty: big.NewInt(1048576), - }, + Genesis: gspec, Ethash: ethash.Config{ PowMode: ethash.ModeFake, }, @@ -257,99 +352,16 @@ func createGQLService(t *testing.T, stack *node.Node) { } // Create some blocks and import them chain, _ := core.GenerateChain(params.AllEthashProtocolChanges, ethBackend.BlockChain().Genesis(), - ethash.NewFaker(), ethBackend.ChainDb(), 10, func(i int, gen *core.BlockGen) {}) - _, err = ethBackend.BlockChain().InsertChain(chain) - if err != nil { - t.Fatalf("could not create import blocks: %v", err) - } - // create gql service - err = New(stack, ethBackend.APIBackend, []string{}, []string{}) - if err != nil { - t.Fatalf("could not create graphql service: %v", err) - } -} - -func createGQLServiceWithTransactions(t *testing.T, stack *node.Node) { - // create backend - key, _ := crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - address := crypto.PubkeyToAddress(key.PublicKey) - funds := big.NewInt(1000000000000000) - dad := common.HexToAddress("0x0000000000000000000000000000000000000dad") - - ethConf := ðconfig.Config{ - Genesis: &core.Genesis{ - Config: params.AllEthashProtocolChanges, - GasLimit: 11500000, - Difficulty: big.NewInt(1048576), - Alloc: core.GenesisAlloc{ - address: {Balance: funds}, - // The address 0xdad sloads 0x00 and 0x01 - dad: { - Code: []byte{ - byte(vm.PC), - byte(vm.PC), - byte(vm.SLOAD), - byte(vm.SLOAD), - }, - Nonce: 0, - Balance: big.NewInt(0), - }, - }, - BaseFee: big.NewInt(params.InitialBaseFee), - }, - Ethash: ethash.Config{ - PowMode: ethash.ModeFake, - }, - NetworkId: 1337, - TrieCleanCache: 5, - TrieCleanCacheJournal: "triecache", - TrieCleanCacheRejournal: 60 * time.Minute, - TrieDirtyCache: 5, - TrieTimeout: 60 * time.Minute, - SnapshotCache: 5, - } - - ethBackend, err := eth.New(stack, ethConf) - if err != nil { - t.Fatalf("could not create eth backend: %v", err) - } - signer := types.LatestSigner(ethConf.Genesis.Config) - - legacyTx, _ := types.SignNewTx(key, signer, &types.LegacyTx{ - Nonce: uint64(0), - To: &dad, - Value: big.NewInt(100), - Gas: 50000, - GasPrice: big.NewInt(params.InitialBaseFee), - }) - envelopTx, _ := types.SignNewTx(key, signer, &types.AccessListTx{ - ChainID: ethConf.Genesis.Config.ChainID, - Nonce: uint64(1), - To: &dad, - Gas: 30000, - GasPrice: big.NewInt(params.InitialBaseFee), - Value: big.NewInt(50), - AccessList: types.AccessList{{ - Address: dad, - StorageKeys: []common.Hash{{0}}, - }}, - }) - - // Create some blocks and import them - chain, _ := core.GenerateChain(params.AllEthashProtocolChanges, ethBackend.BlockChain().Genesis(), - ethash.NewFaker(), ethBackend.ChainDb(), 1, func(i int, b *core.BlockGen) { - b.SetCoinbase(common.Address{1}) - b.AddTx(legacyTx) - b.AddTx(envelopTx) - }) - + ethash.NewFaker(), ethBackend.ChainDb(), genBlocks, genfunc) _, err = ethBackend.BlockChain().InsertChain(chain) if err != nil { t.Fatalf("could not create import blocks: %v", err) } - // create gql service - err = New(stack, ethBackend.APIBackend, []string{}, []string{}) + // Set up handler + filterSystem := filters.NewFilterSystem(ethBackend.APIBackend, filters.Config{}) + handler, err := newHandler(stack, ethBackend.APIBackend, filterSystem, []string{}, []string{}) if err != nil { t.Fatalf("could not create graphql service: %v", err) } + return handler } diff --git a/graphql/service.go b/graphql/service.go index 29d98ad746836..6f6e58335991c 100644 --- a/graphql/service.go +++ b/graphql/service.go @@ -20,6 +20,7 @@ import ( "encoding/json" "net/http" + "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/internal/ethapi" "github.com/ethereum/go-ethereum/node" "github.com/graph-gophers/graphql-go" @@ -52,26 +53,22 @@ func (h handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") w.Write(responseJSON) - } // New constructs a new GraphQL service instance. -func New(stack *node.Node, backend ethapi.Backend, cors, vhosts []string) error { - if backend == nil { - panic("missing backend") - } - // check if http server with given endpoint exists and enable graphQL on it - return newHandler(stack, backend, cors, vhosts) +func New(stack *node.Node, backend ethapi.Backend, filterSystem *filters.FilterSystem, cors, vhosts []string) error { + _, err := newHandler(stack, backend, filterSystem, cors, vhosts) + return err } // newHandler returns a new `http.Handler` that will answer GraphQL queries. // It additionally exports an interactive query browser on the / endpoint. -func newHandler(stack *node.Node, backend ethapi.Backend, cors, vhosts []string) error { - q := Resolver{backend} +func newHandler(stack *node.Node, backend ethapi.Backend, filterSystem *filters.FilterSystem, cors, vhosts []string) (*handler, error) { + q := Resolver{backend, filterSystem} s, err := graphql.ParseSchema(schema, &q) if err != nil { - return err + return nil, err } h := handler{Schema: s} handler := node.NewHTTPHandlerStack(h, cors, vhosts, nil) @@ -80,5 +77,5 @@ func newHandler(stack *node.Node, backend ethapi.Backend, cors, vhosts []string) stack.RegisterHandler("GraphQL", "/graphql", handler) stack.RegisterHandler("GraphQL", "/graphql/", handler) - return nil + return &h, nil } diff --git a/interfaces.go b/interfaces.go index 76c1ef6908f25..eb9af60076e07 100644 --- a/interfaces.go +++ b/interfaces.go @@ -201,6 +201,15 @@ type GasPricer interface { SuggestGasPrice(ctx context.Context) (*big.Int, error) } +// FeeHistory provides recent fee market data that consumers can use to determine +// a reasonable maxPriorityFeePerGas value. +type FeeHistory struct { + OldestBlock *big.Int // block corresponding to first response value + Reward [][]*big.Int // list every txs priority fee per block + BaseFee []*big.Int // list of each block's base fee + GasUsedRatio []float64 // ratio of gas used out of the total available limit +} + // A PendingStateReader provides access to the pending state, which is the result of all // known executable transactions which have not yet been included in the blockchain. It is // commonly used to display the result of ’unconfirmed’ actions (e.g. wallet value diff --git a/internal/build/archive.go b/internal/build/archive.go index 8b3ac23d1d899..c16246070e8c8 100644 --- a/internal/build/archive.go +++ b/internal/build/archive.go @@ -25,6 +25,7 @@ import ( "os" "path/filepath" "strings" + "time" ) type Archive interface { @@ -159,6 +160,7 @@ func (a *TarballArchive) Directory(name string) error { Name: a.dir, Mode: 0755, Typeflag: tar.TypeDir, + ModTime: time.Now(), }) } diff --git a/internal/build/gotool.go b/internal/build/gotool.go index e644b5f69526d..08c8b2ef05fe8 100644 --- a/internal/build/gotool.go +++ b/internal/build/gotool.go @@ -85,7 +85,7 @@ func (g *GoToolchain) goTool(command string, args ...string) *exec.Cmd { if g.Root == "" { g.Root = runtime.GOROOT() } - tool := exec.Command(filepath.Join(g.Root, "bin", "go"), command) + tool := exec.Command(filepath.Join(g.Root, "bin", "go"), command) // nolint: gosec tool.Args = append(tool.Args, args...) tool.Env = append(tool.Env, "GOROOT="+g.Root) diff --git a/internal/build/util.go b/internal/build/util.go index 654349fac307c..9a721e9b83b16 100644 --- a/internal/build/util.go +++ b/internal/build/util.go @@ -29,6 +29,7 @@ import ( "os/exec" "path" "path/filepath" + "strconv" "strings" "text/template" "time" @@ -39,7 +40,7 @@ var DryRunFlag = flag.Bool("n", false, "dry run, don't execute commands") // MustRun executes the given command and exits the host process for // any error. func MustRun(cmd *exec.Cmd) { - fmt.Println(">>>", strings.Join(cmd.Args, " ")) + fmt.Println(">>>", printArgs(cmd.Args)) if !*DryRunFlag { cmd.Stderr = os.Stderr cmd.Stdout = os.Stdout @@ -49,6 +50,20 @@ func MustRun(cmd *exec.Cmd) { } } +func printArgs(args []string) string { + var s strings.Builder + for i, arg := range args { + if i > 0 { + s.WriteByte(' ') + } + if strings.IndexByte(arg, ' ') >= 0 { + arg = strconv.QuoteToASCII(arg) + } + s.WriteString(arg) + } + return s.String() +} + func MustRunCommand(cmd string, args ...string) { MustRun(exec.Command(cmd, args...)) } @@ -121,7 +136,7 @@ func UploadSFTP(identityFile, host, dir string, files []string) error { sftp.Args = append(sftp.Args, "-i", identityFile) } sftp.Args = append(sftp.Args, host) - fmt.Println(">>>", strings.Join(sftp.Args, " ")) + fmt.Println(">>>", printArgs(sftp.Args)) if *DryRunFlag { return nil } diff --git a/internal/debug/flags.go b/internal/debug/flags.go index 3aa990adfb3ab..2082d60df51c0 100644 --- a/internal/debug/flags.go +++ b/internal/debug/flags.go @@ -20,75 +20,88 @@ import ( "fmt" "io" "net/http" - _ "net/http/pprof" + _ "net/http/pprof" // nolint: gosec "os" "runtime" + "github.com/ethereum/go-ethereum/internal/flags" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/metrics/exp" "github.com/fjl/memsize/memsizeui" "github.com/mattn/go-colorable" "github.com/mattn/go-isatty" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) var Memsize memsizeui.Handler var ( - verbosityFlag = cli.IntFlag{ - Name: "verbosity", - Usage: "Logging verbosity: 0=silent, 1=error, 2=warn, 3=info, 4=debug, 5=detail", - Value: 3, - } - vmoduleFlag = cli.StringFlag{ - Name: "vmodule", - Usage: "Per-module verbosity: comma-separated list of = (e.g. eth/*=5,p2p=4)", - Value: "", - } - logjsonFlag = cli.BoolFlag{ - Name: "log.json", - Usage: "Format logs with JSON", - } - backtraceAtFlag = cli.StringFlag{ - Name: "log.backtrace", - Usage: "Request a stack trace at a specific logging statement (e.g. \"block.go:271\")", - Value: "", - } - debugFlag = cli.BoolFlag{ - Name: "log.debug", - Usage: "Prepends log messages with call-site location (file and line number)", - } - pprofFlag = cli.BoolFlag{ - Name: "pprof", - Usage: "Enable the pprof HTTP server", - } - pprofPortFlag = cli.IntFlag{ - Name: "pprof.port", - Usage: "pprof HTTP server listening port", - Value: 6060, - } - pprofAddrFlag = cli.StringFlag{ - Name: "pprof.addr", - Usage: "pprof HTTP server listening interface", - Value: "127.0.0.1", - } - memprofilerateFlag = cli.IntFlag{ - Name: "pprof.memprofilerate", - Usage: "Turn on memory profiling with the given rate", - Value: runtime.MemProfileRate, - } - blockprofilerateFlag = cli.IntFlag{ - Name: "pprof.blockprofilerate", - Usage: "Turn on block profiling with the given rate", - } - cpuprofileFlag = cli.StringFlag{ - Name: "pprof.cpuprofile", - Usage: "Write CPU profile to the given file", - } - traceFlag = cli.StringFlag{ - Name: "trace", - Usage: "Write execution trace to the given file", + verbosityFlag = &cli.IntFlag{ + Name: "verbosity", + Usage: "Logging verbosity: 0=silent, 1=error, 2=warn, 3=info, 4=debug, 5=detail", + Value: 3, + Category: flags.LoggingCategory, + } + vmoduleFlag = &cli.StringFlag{ + Name: "vmodule", + Usage: "Per-module verbosity: comma-separated list of = (e.g. eth/*=5,p2p=4)", + Value: "", + Category: flags.LoggingCategory, + } + logjsonFlag = &cli.BoolFlag{ + Name: "log.json", + Usage: "Format logs with JSON", + Category: flags.LoggingCategory, + } + backtraceAtFlag = &cli.StringFlag{ + Name: "log.backtrace", + Usage: "Request a stack trace at a specific logging statement (e.g. \"block.go:271\")", + Value: "", + Category: flags.LoggingCategory, + } + debugFlag = &cli.BoolFlag{ + Name: "log.debug", + Usage: "Prepends log messages with call-site location (file and line number)", + Category: flags.LoggingCategory, + } + pprofFlag = &cli.BoolFlag{ + Name: "pprof", + Usage: "Enable the pprof HTTP server", + Category: flags.LoggingCategory, + } + pprofPortFlag = &cli.IntFlag{ + Name: "pprof.port", + Usage: "pprof HTTP server listening port", + Value: 6060, + Category: flags.LoggingCategory, + } + pprofAddrFlag = &cli.StringFlag{ + Name: "pprof.addr", + Usage: "pprof HTTP server listening interface", + Value: "127.0.0.1", + Category: flags.LoggingCategory, + } + memprofilerateFlag = &cli.IntFlag{ + Name: "pprof.memprofilerate", + Usage: "Turn on memory profiling with the given rate", + Value: runtime.MemProfileRate, + Category: flags.LoggingCategory, + } + blockprofilerateFlag = &cli.IntFlag{ + Name: "pprof.blockprofilerate", + Usage: "Turn on block profiling with the given rate", + Category: flags.LoggingCategory, + } + cpuprofileFlag = &cli.StringFlag{ + Name: "pprof.cpuprofile", + Usage: "Write CPU profile to the given file", + Category: flags.LoggingCategory, + } + traceFlag = &cli.StringFlag{ + Name: "trace", + Usage: "Write execution trace to the given file", + Category: flags.LoggingCategory, } ) @@ -121,7 +134,7 @@ func init() { func Setup(ctx *cli.Context) error { var ostream log.Handler output := io.Writer(os.Stderr) - if ctx.GlobalBool(logjsonFlag.Name) { + if ctx.Bool(logjsonFlag.Name) { ostream = log.StreamHandler(output, log.JSONFormat()) } else { usecolor := (isatty.IsTerminal(os.Stderr.Fd()) || isatty.IsCygwinTerminal(os.Stderr.Fd())) && os.Getenv("TERM") != "dumb" @@ -133,53 +146,53 @@ func Setup(ctx *cli.Context) error { glogger.SetHandler(ostream) // logging - verbosity := ctx.GlobalInt(verbosityFlag.Name) + verbosity := ctx.Int(verbosityFlag.Name) glogger.Verbosity(log.Lvl(verbosity)) - vmodule := ctx.GlobalString(vmoduleFlag.Name) + vmodule := ctx.String(vmoduleFlag.Name) glogger.Vmodule(vmodule) - debug := ctx.GlobalBool(debugFlag.Name) - if ctx.GlobalIsSet(debugFlag.Name) { - debug = ctx.GlobalBool(debugFlag.Name) + debug := ctx.Bool(debugFlag.Name) + if ctx.IsSet(debugFlag.Name) { + debug = ctx.Bool(debugFlag.Name) } log.PrintOrigins(debug) - backtrace := ctx.GlobalString(backtraceAtFlag.Name) + backtrace := ctx.String(backtraceAtFlag.Name) glogger.BacktraceAt(backtrace) log.Root().SetHandler(glogger) // profiling, tracing runtime.MemProfileRate = memprofilerateFlag.Value - if ctx.GlobalIsSet(memprofilerateFlag.Name) { - runtime.MemProfileRate = ctx.GlobalInt(memprofilerateFlag.Name) + if ctx.IsSet(memprofilerateFlag.Name) { + runtime.MemProfileRate = ctx.Int(memprofilerateFlag.Name) } - blockProfileRate := ctx.GlobalInt(blockprofilerateFlag.Name) + blockProfileRate := ctx.Int(blockprofilerateFlag.Name) Handler.SetBlockProfileRate(blockProfileRate) - if traceFile := ctx.GlobalString(traceFlag.Name); traceFile != "" { + if traceFile := ctx.String(traceFlag.Name); traceFile != "" { if err := Handler.StartGoTrace(traceFile); err != nil { return err } } - if cpuFile := ctx.GlobalString(cpuprofileFlag.Name); cpuFile != "" { + if cpuFile := ctx.String(cpuprofileFlag.Name); cpuFile != "" { if err := Handler.StartCPUProfile(cpuFile); err != nil { return err } } // pprof server - if ctx.GlobalBool(pprofFlag.Name) { - listenHost := ctx.GlobalString(pprofAddrFlag.Name) + if ctx.Bool(pprofFlag.Name) { + listenHost := ctx.String(pprofAddrFlag.Name) - port := ctx.GlobalInt(pprofPortFlag.Name) + port := ctx.Int(pprofPortFlag.Name) address := fmt.Sprintf("%s:%d", listenHost, port) // This context value ("metrics.addr") represents the utils.MetricsHTTPFlag.Name. // It cannot be imported because it will cause a cyclical dependency. - StartPProf(address, !ctx.GlobalIsSet("metrics.addr")) + StartPProf(address, !ctx.IsSet("metrics.addr")) } return nil } diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 1b7c177867477..e6740942d859e 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -48,19 +48,18 @@ import ( "github.com/tyler-smith/go-bip39" ) -// PublicEthereumAPI provides an API to access Ethereum related information. -// It offers only methods that operate on public data that is freely available to anyone. -type PublicEthereumAPI struct { +// EthereumAPI provides an API to access Ethereum related information. +type EthereumAPI struct { b Backend } -// NewPublicEthereumAPI creates a new Ethereum protocol API. -func NewPublicEthereumAPI(b Backend) *PublicEthereumAPI { - return &PublicEthereumAPI{b} +// NewEthereumAPI creates a new Ethereum protocol API. +func NewEthereumAPI(b Backend) *EthereumAPI { + return &EthereumAPI{b} } // GasPrice returns a suggestion for a gas price for legacy transactions. -func (s *PublicEthereumAPI) GasPrice(ctx context.Context) (*hexutil.Big, error) { +func (s *EthereumAPI) GasPrice(ctx context.Context) (*hexutil.Big, error) { tipcap, err := s.b.SuggestGasTipCap(ctx) if err != nil { return nil, err @@ -72,7 +71,7 @@ func (s *PublicEthereumAPI) GasPrice(ctx context.Context) (*hexutil.Big, error) } // MaxPriorityFeePerGas returns a suggestion for a gas tip cap for dynamic fee transactions. -func (s *PublicEthereumAPI) MaxPriorityFeePerGas(ctx context.Context) (*hexutil.Big, error) { +func (s *EthereumAPI) MaxPriorityFeePerGas(ctx context.Context) (*hexutil.Big, error) { tipcap, err := s.b.SuggestGasTipCap(ctx) if err != nil { return nil, err @@ -87,7 +86,8 @@ type feeHistoryResult struct { GasUsedRatio []float64 `json:"gasUsedRatio"` } -func (s *PublicEthereumAPI) FeeHistory(ctx context.Context, blockCount rpc.DecimalOrHex, lastBlock rpc.BlockNumber, rewardPercentiles []float64) (*feeHistoryResult, error) { +// FeeHistory returns the fee market history. +func (s *EthereumAPI) FeeHistory(ctx context.Context, blockCount rpc.DecimalOrHex, lastBlock rpc.BlockNumber, rewardPercentiles []float64) (*feeHistoryResult, error) { oldest, reward, baseFee, gasUsed, err := s.b.FeeHistory(ctx, int(blockCount), lastBlock, rewardPercentiles) if err != nil { return nil, err @@ -121,7 +121,7 @@ func (s *PublicEthereumAPI) FeeHistory(ctx context.Context, blockCount rpc.Decim // - highestBlock: block number of the highest block header this node has received from peers // - pulledStates: number of state entries processed until now // - knownStates: number of known state entries that still need to be pulled -func (s *PublicEthereumAPI) Syncing() (interface{}, error) { +func (s *EthereumAPI) Syncing() (interface{}, error) { progress := s.b.SyncProgress() // Return not syncing if the synchronisation already completed @@ -148,18 +148,18 @@ func (s *PublicEthereumAPI) Syncing() (interface{}, error) { }, nil } -// PublicTxPoolAPI offers and API for the transaction pool. It only operates on data that is non confidential. -type PublicTxPoolAPI struct { +// TxPoolAPI offers and API for the transaction pool. It only operates on data that is non confidential. +type TxPoolAPI struct { b Backend } -// NewPublicTxPoolAPI creates a new tx pool service that gives information about the transaction pool. -func NewPublicTxPoolAPI(b Backend) *PublicTxPoolAPI { - return &PublicTxPoolAPI{b} +// NewTxPoolAPI creates a new tx pool service that gives information about the transaction pool. +func NewTxPoolAPI(b Backend) *TxPoolAPI { + return &TxPoolAPI{b} } // Content returns the transactions contained within the transaction pool. -func (s *PublicTxPoolAPI) Content() map[string]map[string]map[string]*RPCTransaction { +func (s *TxPoolAPI) Content() map[string]map[string]map[string]*RPCTransaction { content := map[string]map[string]map[string]*RPCTransaction{ "pending": make(map[string]map[string]*RPCTransaction), "queued": make(map[string]map[string]*RPCTransaction), @@ -186,7 +186,7 @@ func (s *PublicTxPoolAPI) Content() map[string]map[string]map[string]*RPCTransac } // ContentFrom returns the transactions contained within the transaction pool. -func (s *PublicTxPoolAPI) ContentFrom(addr common.Address) map[string]map[string]*RPCTransaction { +func (s *TxPoolAPI) ContentFrom(addr common.Address) map[string]map[string]*RPCTransaction { content := make(map[string]map[string]*RPCTransaction, 2) pending, queue := s.b.TxPoolContentFrom(addr) curHeader := s.b.CurrentHeader() @@ -209,7 +209,7 @@ func (s *PublicTxPoolAPI) ContentFrom(addr common.Address) map[string]map[string } // Status returns the number of pending and queued transaction in the pool. -func (s *PublicTxPoolAPI) Status() map[string]hexutil.Uint { +func (s *TxPoolAPI) Status() map[string]hexutil.Uint { pending, queue := s.b.Stats() return map[string]hexutil.Uint{ "pending": hexutil.Uint(pending), @@ -219,7 +219,7 @@ func (s *PublicTxPoolAPI) Status() map[string]hexutil.Uint { // Inspect retrieves the content of the transaction pool and flattens it into an // easily inspectable list. -func (s *PublicTxPoolAPI) Inspect() map[string]map[string]map[string]string { +func (s *TxPoolAPI) Inspect() map[string]map[string]map[string]string { content := map[string]map[string]map[string]string{ "pending": make(map[string]map[string]string), "queued": make(map[string]map[string]string), @@ -252,34 +252,34 @@ func (s *PublicTxPoolAPI) Inspect() map[string]map[string]map[string]string { return content } -// PublicAccountAPI provides an API to access accounts managed by this node. +// EthereumAccountAPI provides an API to access accounts managed by this node. // It offers only methods that can retrieve accounts. -type PublicAccountAPI struct { +type EthereumAccountAPI struct { am *accounts.Manager } -// NewPublicAccountAPI creates a new PublicAccountAPI. -func NewPublicAccountAPI(am *accounts.Manager) *PublicAccountAPI { - return &PublicAccountAPI{am: am} +// NewEthereumAccountAPI creates a new EthereumAccountAPI. +func NewEthereumAccountAPI(am *accounts.Manager) *EthereumAccountAPI { + return &EthereumAccountAPI{am: am} } -// Accounts returns the collection of accounts this node manages -func (s *PublicAccountAPI) Accounts() []common.Address { +// Accounts returns the collection of accounts this node manages. +func (s *EthereumAccountAPI) Accounts() []common.Address { return s.am.Accounts() } -// PrivateAccountAPI provides an API to access accounts managed by this node. +// PersonalAccountAPI provides an API to access accounts managed by this node. // It offers methods to create, (un)lock en list accounts. Some methods accept // passwords and are therefore considered private by default. -type PrivateAccountAPI struct { +type PersonalAccountAPI struct { am *accounts.Manager nonceLock *AddrLocker b Backend } -// NewPrivateAccountAPI create a new PrivateAccountAPI. -func NewPrivateAccountAPI(b Backend, nonceLock *AddrLocker) *PrivateAccountAPI { - return &PrivateAccountAPI{ +// NewPersonalAccountAPI create a new PersonalAccountAPI. +func NewPersonalAccountAPI(b Backend, nonceLock *AddrLocker) *PersonalAccountAPI { + return &PersonalAccountAPI{ am: b.AccountManager(), nonceLock: nonceLock, b: b, @@ -287,7 +287,7 @@ func NewPrivateAccountAPI(b Backend, nonceLock *AddrLocker) *PrivateAccountAPI { } // ListAccounts will return a list of addresses for accounts this node manages. -func (s *PrivateAccountAPI) ListAccounts() []common.Address { +func (s *PersonalAccountAPI) ListAccounts() []common.Address { return s.am.Accounts() } @@ -301,7 +301,7 @@ type rawWallet struct { } // ListWallets will return a list of wallets this node manages. -func (s *PrivateAccountAPI) ListWallets() []rawWallet { +func (s *PersonalAccountAPI) ListWallets() []rawWallet { wallets := make([]rawWallet, 0) // return [] instead of nil if empty for _, wallet := range s.am.Wallets() { status, failure := wallet.Status() @@ -323,7 +323,7 @@ func (s *PrivateAccountAPI) ListWallets() []rawWallet { // connection and attempting to authenticate via the provided passphrase. Note, // the method may return an extra challenge requiring a second open (e.g. the // Trezor PIN matrix challenge). -func (s *PrivateAccountAPI) OpenWallet(url string, passphrase *string) error { +func (s *PersonalAccountAPI) OpenWallet(url string, passphrase *string) error { wallet, err := s.am.Wallet(url) if err != nil { return err @@ -337,7 +337,7 @@ func (s *PrivateAccountAPI) OpenWallet(url string, passphrase *string) error { // DeriveAccount requests a HD wallet to derive a new account, optionally pinning // it for later reuse. -func (s *PrivateAccountAPI) DeriveAccount(url string, path string, pin *bool) (accounts.Account, error) { +func (s *PersonalAccountAPI) DeriveAccount(url string, path string, pin *bool) (accounts.Account, error) { wallet, err := s.am.Wallet(url) if err != nil { return accounts.Account{}, err @@ -353,7 +353,7 @@ func (s *PrivateAccountAPI) DeriveAccount(url string, path string, pin *bool) (a } // NewAccount will create a new account and returns the address for the new account. -func (s *PrivateAccountAPI) NewAccount(password string) (common.Address, error) { +func (s *PersonalAccountAPI) NewAccount(password string) (common.Address, error) { ks, err := fetchKeystore(s.am) if err != nil { return common.Address{}, err @@ -378,7 +378,7 @@ func fetchKeystore(am *accounts.Manager) (*keystore.KeyStore, error) { // ImportRawKey stores the given hex encoded ECDSA key into the key directory, // encrypting it with the passphrase. -func (s *PrivateAccountAPI) ImportRawKey(privkey string, password string) (common.Address, error) { +func (s *PersonalAccountAPI) ImportRawKey(privkey string, password string) (common.Address, error) { key, err := crypto.HexToECDSA(privkey) if err != nil { return common.Address{}, err @@ -394,7 +394,7 @@ func (s *PrivateAccountAPI) ImportRawKey(privkey string, password string) (commo // UnlockAccount will unlock the account associated with the given address with // the given password for duration seconds. If duration is nil it will use a // default of 300 seconds. It returns an indication if the account was unlocked. -func (s *PrivateAccountAPI) UnlockAccount(ctx context.Context, addr common.Address, password string, duration *uint64) (bool, error) { +func (s *PersonalAccountAPI) UnlockAccount(ctx context.Context, addr common.Address, password string, duration *uint64) (bool, error) { // When the API is exposed by external RPC(http, ws etc), unless the user // explicitly specifies to allow the insecure account unlocking, otherwise // it is disabled. @@ -423,7 +423,7 @@ func (s *PrivateAccountAPI) UnlockAccount(ctx context.Context, addr common.Addre } // LockAccount will lock the account associated with the given address when it's unlocked. -func (s *PrivateAccountAPI) LockAccount(addr common.Address) bool { +func (s *PersonalAccountAPI) LockAccount(addr common.Address) bool { if ks, err := fetchKeystore(s.am); err == nil { return ks.Lock(addr) == nil } @@ -433,7 +433,7 @@ func (s *PrivateAccountAPI) LockAccount(addr common.Address) bool { // signTransaction sets defaults and signs the given transaction // NOTE: the caller needs to ensure that the nonceLock is held, if applicable, // and release it after the transaction has been submitted to the tx pool -func (s *PrivateAccountAPI) signTransaction(ctx context.Context, args *TransactionArgs, passwd string) (*types.Transaction, error) { +func (s *PersonalAccountAPI) signTransaction(ctx context.Context, args *TransactionArgs, passwd string) (*types.Transaction, error) { // Look up the wallet containing the requested signer account := accounts.Account{Address: args.from()} wallet, err := s.am.Find(account) @@ -453,7 +453,7 @@ func (s *PrivateAccountAPI) signTransaction(ctx context.Context, args *Transacti // SendTransaction will create a transaction from the given arguments and // tries to sign it with the key associated with args.From. If the given // passwd isn't able to decrypt the key it fails. -func (s *PrivateAccountAPI) SendTransaction(ctx context.Context, args TransactionArgs, passwd string) (common.Hash, error) { +func (s *PersonalAccountAPI) SendTransaction(ctx context.Context, args TransactionArgs, passwd string) (common.Hash, error) { if args.Nonce == nil { // Hold the addresse's mutex around signing to prevent concurrent assignment of // the same nonce to multiple accounts. @@ -472,7 +472,7 @@ func (s *PrivateAccountAPI) SendTransaction(ctx context.Context, args Transactio // tries to sign it with the key associated with args.From. If the given passwd isn't // able to decrypt the key it fails. The transaction is returned in RLP-form, not broadcast // to other nodes -func (s *PrivateAccountAPI) SignTransaction(ctx context.Context, args TransactionArgs, passwd string) (*SignTransactionResult, error) { +func (s *PersonalAccountAPI) SignTransaction(ctx context.Context, args TransactionArgs, passwd string) (*SignTransactionResult, error) { // No need to obtain the noncelock mutex, since we won't be sending this // tx into the transaction pool, but right back to the user if args.From == nil { @@ -505,7 +505,7 @@ func (s *PrivateAccountAPI) SignTransaction(ctx context.Context, args Transactio } // Sign calculates an Ethereum ECDSA signature for: -// keccack256("\x19Ethereum Signed Message:\n" + len(message) + message)) +// keccak256("\x19Ethereum Signed Message:\n" + len(message) + message)) // // Note, the produced signature conforms to the secp256k1 curve R, S and V values, // where the V value will be 27 or 28 for legacy reasons. @@ -513,7 +513,7 @@ func (s *PrivateAccountAPI) SignTransaction(ctx context.Context, args Transactio // The key used to calculate the signature is decrypted with the given password. // // https://github.com/ethereum/go-ethereum/wiki/Management-APIs#personal_sign -func (s *PrivateAccountAPI) Sign(ctx context.Context, data hexutil.Bytes, addr common.Address, passwd string) (hexutil.Bytes, error) { +func (s *PersonalAccountAPI) Sign(ctx context.Context, data hexutil.Bytes, addr common.Address, passwd string) (hexutil.Bytes, error) { // Look up the wallet containing the requested signer account := accounts.Account{Address: addr} @@ -541,7 +541,7 @@ func (s *PrivateAccountAPI) Sign(ctx context.Context, data hexutil.Bytes, addr c // the V value must be 27 or 28 for legacy reasons. // // https://github.com/ethereum/go-ethereum/wiki/Management-APIs#personal_ecRecover -func (s *PrivateAccountAPI) EcRecover(ctx context.Context, data, sig hexutil.Bytes) (common.Address, error) { +func (s *PersonalAccountAPI) EcRecover(ctx context.Context, data, sig hexutil.Bytes) (common.Address, error) { if len(sig) != crypto.SignatureLength { return common.Address{}, fmt.Errorf("signature must be %d bytes long", crypto.SignatureLength) } @@ -557,14 +557,8 @@ func (s *PrivateAccountAPI) EcRecover(ctx context.Context, data, sig hexutil.Byt return crypto.PubkeyToAddress(*rpk), nil } -// SignAndSendTransaction was renamed to SendTransaction. This method is deprecated -// and will be removed in the future. It primary goal is to give clients time to update. -func (s *PrivateAccountAPI) SignAndSendTransaction(ctx context.Context, args TransactionArgs, passwd string) (common.Hash, error) { - return s.SendTransaction(ctx, args, passwd) -} - // InitializeWallet initializes a new wallet at the provided URL, by generating and returning a new private key. -func (s *PrivateAccountAPI) InitializeWallet(ctx context.Context, url string) (string, error) { +func (s *PersonalAccountAPI) InitializeWallet(ctx context.Context, url string) (string, error) { wallet, err := s.am.Wallet(url) if err != nil { return "", err @@ -591,7 +585,7 @@ func (s *PrivateAccountAPI) InitializeWallet(ctx context.Context, url string) (s } // Unpair deletes a pairing between wallet and geth. -func (s *PrivateAccountAPI) Unpair(ctx context.Context, url string, pin string) error { +func (s *PersonalAccountAPI) Unpair(ctx context.Context, url string, pin string) error { wallet, err := s.am.Wallet(url) if err != nil { return err @@ -605,28 +599,28 @@ func (s *PrivateAccountAPI) Unpair(ctx context.Context, url string, pin string) } } -// PublicBlockChainAPI provides an API to access the Ethereum blockchain. -// It offers only methods that operate on public data that is freely available to anyone. -type PublicBlockChainAPI struct { +// BlockChainAPI provides an API to access Ethereum blockchain data. +type BlockChainAPI struct { b Backend } -// NewPublicBlockChainAPI creates a new Ethereum blockchain API. -func NewPublicBlockChainAPI(b Backend) *PublicBlockChainAPI { - return &PublicBlockChainAPI{b} +// NewBlockChainAPI creates a new Ethereum blockchain API. +func NewBlockChainAPI(b Backend) *BlockChainAPI { + return &BlockChainAPI{b} } -// ChainId is the EIP-155 replay-protection chain id for the current ethereum chain config. -func (api *PublicBlockChainAPI) ChainId() (*hexutil.Big, error) { - // if current block is at or past the EIP-155 replay-protection fork block, return chainID from config - if config := api.b.ChainConfig(); config.IsEIP155(api.b.CurrentBlock().Number()) { - return (*hexutil.Big)(config.ChainID), nil - } - return nil, fmt.Errorf("chain not synced beyond EIP-155 replay-protection fork block") +// ChainId is the EIP-155 replay-protection chain id for the current Ethereum chain config. +// +// Note, this method does not conform to EIP-695 because the configured chain ID is always +// returned, regardless of the current head block. We used to return an error when the chain +// wasn't synced up to a block where EIP-155 is enabled, but this behavior caused issues +// in CL clients. +func (api *BlockChainAPI) ChainId() *hexutil.Big { + return (*hexutil.Big)(api.b.ChainConfig().ChainID) } // BlockNumber returns the block number of the chain head. -func (s *PublicBlockChainAPI) BlockNumber() hexutil.Uint64 { +func (s *BlockChainAPI) BlockNumber() hexutil.Uint64 { header, _ := s.b.HeaderByNumber(context.Background(), rpc.LatestBlockNumber) // latest header should always be available return hexutil.Uint64(header.Number.Uint64()) } @@ -634,7 +628,7 @@ func (s *PublicBlockChainAPI) BlockNumber() hexutil.Uint64 { // GetBalance returns the amount of wei for the given address in the state of the // given block number. The rpc.LatestBlockNumber and rpc.PendingBlockNumber meta // block numbers are also allowed. -func (s *PublicBlockChainAPI) GetBalance(ctx context.Context, address common.Address, blockNrOrHash rpc.BlockNumberOrHash) (*hexutil.Big, error) { +func (s *BlockChainAPI) GetBalance(ctx context.Context, address common.Address, blockNrOrHash rpc.BlockNumberOrHash) (*hexutil.Big, error) { state, _, err := s.b.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash) if state == nil || err != nil { return nil, err @@ -660,7 +654,7 @@ type StorageResult struct { } // GetProof returns the Merkle-proof for a given account and optionally some storage keys. -func (s *PublicBlockChainAPI) GetProof(ctx context.Context, address common.Address, storageKeys []string, blockNrOrHash rpc.BlockNumberOrHash) (*AccountResult, error) { +func (s *BlockChainAPI) GetProof(ctx context.Context, address common.Address, storageKeys []string, blockNrOrHash rpc.BlockNumberOrHash) (*AccountResult, error) { state, _, err := s.b.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash) if state == nil || err != nil { return nil, err @@ -712,7 +706,7 @@ func (s *PublicBlockChainAPI) GetProof(ctx context.Context, address common.Addre // GetHeaderByNumber returns the requested canonical block header. // * When blockNr is -1 the chain head is returned. // * When blockNr is -2 the pending chain head is returned. -func (s *PublicBlockChainAPI) GetHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (map[string]interface{}, error) { +func (s *BlockChainAPI) GetHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (map[string]interface{}, error) { header, err := s.b.HeaderByNumber(ctx, number) if header != nil && err == nil { response := s.rpcMarshalHeader(ctx, header) @@ -728,7 +722,7 @@ func (s *PublicBlockChainAPI) GetHeaderByNumber(ctx context.Context, number rpc. } // GetHeaderByHash returns the requested header by hash. -func (s *PublicBlockChainAPI) GetHeaderByHash(ctx context.Context, hash common.Hash) map[string]interface{} { +func (s *BlockChainAPI) GetHeaderByHash(ctx context.Context, hash common.Hash) map[string]interface{} { header, _ := s.b.HeaderByHash(ctx, hash) if header != nil { return s.rpcMarshalHeader(ctx, header) @@ -741,7 +735,7 @@ func (s *PublicBlockChainAPI) GetHeaderByHash(ctx context.Context, hash common.H // * When blockNr is -2 the pending chain head is returned. // * When fullTx is true all transactions in the block are returned, otherwise // only the transaction hash is returned. -func (s *PublicBlockChainAPI) GetBlockByNumber(ctx context.Context, number rpc.BlockNumber, fullTx bool) (map[string]interface{}, error) { +func (s *BlockChainAPI) GetBlockByNumber(ctx context.Context, number rpc.BlockNumber, fullTx bool) (map[string]interface{}, error) { block, err := s.b.BlockByNumber(ctx, number) if block != nil && err == nil { response, err := s.rpcMarshalBlock(ctx, block, true, fullTx) @@ -758,7 +752,7 @@ func (s *PublicBlockChainAPI) GetBlockByNumber(ctx context.Context, number rpc.B // GetBlockByHash returns the requested block. When fullTx is true all transactions in the block are returned in full // detail, otherwise only the transaction hash is returned. -func (s *PublicBlockChainAPI) GetBlockByHash(ctx context.Context, hash common.Hash, fullTx bool) (map[string]interface{}, error) { +func (s *BlockChainAPI) GetBlockByHash(ctx context.Context, hash common.Hash, fullTx bool) (map[string]interface{}, error) { block, err := s.b.BlockByHash(ctx, hash) if block != nil { return s.rpcMarshalBlock(ctx, block, true, fullTx) @@ -767,7 +761,7 @@ func (s *PublicBlockChainAPI) GetBlockByHash(ctx context.Context, hash common.Ha } // GetUncleByBlockNumberAndIndex returns the uncle block for the given block hash and index. -func (s *PublicBlockChainAPI) GetUncleByBlockNumberAndIndex(ctx context.Context, blockNr rpc.BlockNumber, index hexutil.Uint) (map[string]interface{}, error) { +func (s *BlockChainAPI) GetUncleByBlockNumberAndIndex(ctx context.Context, blockNr rpc.BlockNumber, index hexutil.Uint) (map[string]interface{}, error) { block, err := s.b.BlockByNumber(ctx, blockNr) if block != nil { uncles := block.Uncles() @@ -782,7 +776,7 @@ func (s *PublicBlockChainAPI) GetUncleByBlockNumberAndIndex(ctx context.Context, } // GetUncleByBlockHashAndIndex returns the uncle block for the given block hash and index. -func (s *PublicBlockChainAPI) GetUncleByBlockHashAndIndex(ctx context.Context, blockHash common.Hash, index hexutil.Uint) (map[string]interface{}, error) { +func (s *BlockChainAPI) GetUncleByBlockHashAndIndex(ctx context.Context, blockHash common.Hash, index hexutil.Uint) (map[string]interface{}, error) { block, err := s.b.BlockByHash(ctx, blockHash) if block != nil { uncles := block.Uncles() @@ -797,7 +791,7 @@ func (s *PublicBlockChainAPI) GetUncleByBlockHashAndIndex(ctx context.Context, b } // GetUncleCountByBlockNumber returns number of uncles in the block for the given block number -func (s *PublicBlockChainAPI) GetUncleCountByBlockNumber(ctx context.Context, blockNr rpc.BlockNumber) *hexutil.Uint { +func (s *BlockChainAPI) GetUncleCountByBlockNumber(ctx context.Context, blockNr rpc.BlockNumber) *hexutil.Uint { if block, _ := s.b.BlockByNumber(ctx, blockNr); block != nil { n := hexutil.Uint(len(block.Uncles())) return &n @@ -806,7 +800,7 @@ func (s *PublicBlockChainAPI) GetUncleCountByBlockNumber(ctx context.Context, bl } // GetUncleCountByBlockHash returns number of uncles in the block for the given block hash -func (s *PublicBlockChainAPI) GetUncleCountByBlockHash(ctx context.Context, blockHash common.Hash) *hexutil.Uint { +func (s *BlockChainAPI) GetUncleCountByBlockHash(ctx context.Context, blockHash common.Hash) *hexutil.Uint { if block, _ := s.b.BlockByHash(ctx, blockHash); block != nil { n := hexutil.Uint(len(block.Uncles())) return &n @@ -815,7 +809,7 @@ func (s *PublicBlockChainAPI) GetUncleCountByBlockHash(ctx context.Context, bloc } // GetCode returns the code stored at the given address in the state for the given block number. -func (s *PublicBlockChainAPI) GetCode(ctx context.Context, address common.Address, blockNrOrHash rpc.BlockNumberOrHash) (hexutil.Bytes, error) { +func (s *BlockChainAPI) GetCode(ctx context.Context, address common.Address, blockNrOrHash rpc.BlockNumberOrHash) (hexutil.Bytes, error) { state, _, err := s.b.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash) if state == nil || err != nil { return nil, err @@ -827,7 +821,7 @@ func (s *PublicBlockChainAPI) GetCode(ctx context.Context, address common.Addres // GetStorageAt returns the storage from the state at the given address, key and // block number. The rpc.LatestBlockNumber and rpc.PendingBlockNumber meta block // numbers are also allowed. -func (s *PublicBlockChainAPI) GetStorageAt(ctx context.Context, address common.Address, key string, blockNrOrHash rpc.BlockNumberOrHash) (hexutil.Bytes, error) { +func (s *BlockChainAPI) GetStorageAt(ctx context.Context, address common.Address, key string, blockNrOrHash rpc.BlockNumberOrHash) (hexutil.Bytes, error) { state, _, err := s.b.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash) if state == nil || err != nil { return nil, err @@ -888,6 +882,45 @@ func (diff *StateOverride) Apply(state *state.StateDB) error { return nil } +// BlockOverrides is a set of header fields to override. +type BlockOverrides struct { + Number *hexutil.Big + Difficulty *hexutil.Big + Time *hexutil.Big + GasLimit *hexutil.Uint64 + Coinbase *common.Address + Random *common.Hash + BaseFee *hexutil.Big +} + +// Apply overrides the given header fields into the given block context. +func (diff *BlockOverrides) Apply(blockCtx *vm.BlockContext) { + if diff == nil { + return + } + if diff.Number != nil { + blockCtx.BlockNumber = diff.Number.ToInt() + } + if diff.Difficulty != nil { + blockCtx.Difficulty = diff.Difficulty.ToInt() + } + if diff.Time != nil { + blockCtx.Time = diff.Time.ToInt() + } + if diff.GasLimit != nil { + blockCtx.GasLimit = uint64(*diff.GasLimit) + } + if diff.Coinbase != nil { + blockCtx.Coinbase = *diff.Coinbase + } + if diff.Random != nil { + blockCtx.Random = diff.Random + } + if diff.BaseFee != nil { + blockCtx.BaseFee = diff.BaseFee.ToInt() + } +} + func DoCall(ctx context.Context, b Backend, args TransactionArgs, blockNrOrHash rpc.BlockNumberOrHash, overrides *StateOverride, timeout time.Duration, globalGasCap uint64) (*core.ExecutionResult, error) { defer func(start time.Time) { log.Debug("Executing EVM call finished", "runtime", time.Since(start)) }(time.Now()) @@ -955,7 +988,7 @@ func newRevertError(result *core.ExecutionResult) *revertError { } } -// revertError is an API error that encompassas an EVM revertal with JSON error +// revertError is an API error that encompasses an EVM revertal with JSON error // code and a binary data blob. type revertError struct { error @@ -979,7 +1012,7 @@ func (e *revertError) ErrorData() interface{} { // // Note, this function doesn't make and changes in the state/blockchain and is // useful to execute and retrieve values. -func (s *PublicBlockChainAPI) Call(ctx context.Context, args TransactionArgs, blockNrOrHash rpc.BlockNumberOrHash, overrides *StateOverride) (hexutil.Bytes, error) { +func (s *BlockChainAPI) Call(ctx context.Context, args TransactionArgs, blockNrOrHash rpc.BlockNumberOrHash, overrides *StateOverride) (hexutil.Bytes, error) { result, err := DoCall(ctx, s.b, args, blockNrOrHash, overrides, s.b.RPCEVMTimeout(), s.b.RPCGasCap()) if err != nil { return nil, err @@ -1113,7 +1146,7 @@ func DoEstimateGas(ctx context.Context, b Backend, args TransactionArgs, blockNr // EstimateGas returns an estimate of the amount of gas needed to execute the // given transaction against the current pending block. -func (s *PublicBlockChainAPI) EstimateGas(ctx context.Context, args TransactionArgs, blockNrOrHash *rpc.BlockNumberOrHash) (hexutil.Uint64, error) { +func (s *BlockChainAPI) EstimateGas(ctx context.Context, args TransactionArgs, blockNrOrHash *rpc.BlockNumberOrHash) (hexutil.Uint64, error) { bNrOrHash := rpc.BlockNumberOrHashWithNumber(rpc.PendingBlockNumber) if blockNrOrHash != nil { bNrOrHash = *blockNrOrHash @@ -1187,16 +1220,16 @@ func RPCMarshalBlock(block *types.Block, inclTx bool, fullTx bool, config *param } // rpcMarshalHeader uses the generalized output filler, then adds the total difficulty field, which requires -// a `PublicBlockchainAPI`. -func (s *PublicBlockChainAPI) rpcMarshalHeader(ctx context.Context, header *types.Header) map[string]interface{} { +// a `BlockchainAPI`. +func (s *BlockChainAPI) rpcMarshalHeader(ctx context.Context, header *types.Header) map[string]interface{} { fields := RPCMarshalHeader(header) fields["totalDifficulty"] = (*hexutil.Big)(s.b.GetTd(ctx, header.Hash())) return fields } // rpcMarshalBlock uses the generalized output filler, then adds the total difficulty field, which requires -// a `PublicBlockchainAPI`. -func (s *PublicBlockChainAPI) rpcMarshalBlock(ctx context.Context, b *types.Block, inclTx bool, fullTx bool) (map[string]interface{}, error) { +// a `BlockchainAPI`. +func (s *BlockChainAPI) rpcMarshalBlock(ctx context.Context, b *types.Block, inclTx bool, fullTx bool) (map[string]interface{}, error) { fields, err := RPCMarshalBlock(b, inclTx, fullTx, s.b.ChainConfig()) if err != nil { return nil, err @@ -1233,7 +1266,7 @@ type RPCTransaction struct { // newRPCTransaction returns a transaction that will serialize to the RPC // representation, with the given location metadata set (if available). func newRPCTransaction(tx *types.Transaction, blockHash common.Hash, blockNumber uint64, index uint64, baseFee *big.Int, config *params.ChainConfig) *RPCTransaction { - signer := types.MakeSigner(config, big.NewInt(0).SetUint64(blockNumber)) + signer := types.MakeSigner(config, new(big.Int).SetUint64(blockNumber)) from, _ := types.Sender(signer, tx) v, r, s := tx.RawSignatureValues() result := &RPCTransaction{ @@ -1256,6 +1289,11 @@ func newRPCTransaction(tx *types.Transaction, blockHash common.Hash, blockNumber result.TransactionIndex = (*hexutil.Uint64)(&index) } switch tx.Type() { + case types.LegacyTxType: + // if a legacy transaction has an EIP-155 chain id, include it explicitly + if id := tx.ChainId(); id.Sign() != 0 { + result.ChainID = (*hexutil.Big)(id) + } case types.AccessListTxType: al := tx.AccessList() result.Accesses = &al @@ -1329,7 +1367,7 @@ type accessListResult struct { // CreateAccessList creates a EIP-2930 type AccessList for the given transaction. // Reexec and BlockNrOrHash can be specified to create the accessList on top of a certain state. -func (s *PublicBlockChainAPI) CreateAccessList(ctx context.Context, args TransactionArgs, blockNrOrHash *rpc.BlockNumberOrHash) (*accessListResult, error) { +func (s *BlockChainAPI) CreateAccessList(ctx context.Context, args TransactionArgs, blockNrOrHash *rpc.BlockNumberOrHash) (*accessListResult, error) { bNrOrHash := rpc.BlockNumberOrHashWithNumber(rpc.PendingBlockNumber) if blockNrOrHash != nil { bNrOrHash = *blockNrOrHash @@ -1354,9 +1392,11 @@ func AccessList(ctx context.Context, b Backend, blockNrOrHash rpc.BlockNumberOrH if db == nil || err != nil { return nil, 0, nil, err } - // If the gas amount is not set, extract this as it will depend on access - // lists and we'll need to reestimate every time - nogas := args.Gas == nil + // If the gas amount is not set, default to RPC gas cap. + if args.Gas == nil { + tmp := hexutil.Uint64(b.RPCGasCap()) + args.Gas = &tmp + } // Ensure any missing fields are filled, extract the recipient and input data if err := args.setDefaults(ctx, b); err != nil { @@ -1382,15 +1422,6 @@ func AccessList(ctx context.Context, b Backend, blockNrOrHash rpc.BlockNumberOrH accessList := prevTracer.AccessList() log.Trace("Creating access list", "input", accessList) - // If no gas amount was specified, each unique access list needs it's own - // gas calculation. This is quite expensive, but we need to be accurate - // and it's convered by the sender only anyway. - if nogas { - args.Gas = nil - if err := args.setDefaults(ctx, b); err != nil { - return nil, 0, nil, err // shouldn't happen, just in case - } - } // Copy the original db so we don't modify it statedb := db.Copy() // Set the accesslist to the last al @@ -1418,23 +1449,23 @@ func AccessList(ctx context.Context, b Backend, blockNrOrHash rpc.BlockNumberOrH } } -// PublicTransactionPoolAPI exposes methods for the RPC interface -type PublicTransactionPoolAPI struct { +// TransactionAPI exposes methods for reading and creating transaction data. +type TransactionAPI struct { b Backend nonceLock *AddrLocker signer types.Signer } -// NewPublicTransactionPoolAPI creates a new RPC service with methods specific for the transaction pool. -func NewPublicTransactionPoolAPI(b Backend, nonceLock *AddrLocker) *PublicTransactionPoolAPI { +// NewTransactionAPI creates a new RPC service with methods for interacting with transactions. +func NewTransactionAPI(b Backend, nonceLock *AddrLocker) *TransactionAPI { // The signer used by the API should always be the 'latest' known one because we expect // signers to be backwards-compatible with old transactions. signer := types.LatestSigner(b.ChainConfig()) - return &PublicTransactionPoolAPI{b, nonceLock, signer} + return &TransactionAPI{b, nonceLock, signer} } // GetBlockTransactionCountByNumber returns the number of transactions in the block with the given block number. -func (s *PublicTransactionPoolAPI) GetBlockTransactionCountByNumber(ctx context.Context, blockNr rpc.BlockNumber) *hexutil.Uint { +func (s *TransactionAPI) GetBlockTransactionCountByNumber(ctx context.Context, blockNr rpc.BlockNumber) *hexutil.Uint { if block, _ := s.b.BlockByNumber(ctx, blockNr); block != nil { n := hexutil.Uint(len(block.Transactions())) return &n @@ -1443,7 +1474,7 @@ func (s *PublicTransactionPoolAPI) GetBlockTransactionCountByNumber(ctx context. } // GetBlockTransactionCountByHash returns the number of transactions in the block with the given hash. -func (s *PublicTransactionPoolAPI) GetBlockTransactionCountByHash(ctx context.Context, blockHash common.Hash) *hexutil.Uint { +func (s *TransactionAPI) GetBlockTransactionCountByHash(ctx context.Context, blockHash common.Hash) *hexutil.Uint { if block, _ := s.b.BlockByHash(ctx, blockHash); block != nil { n := hexutil.Uint(len(block.Transactions())) return &n @@ -1452,7 +1483,7 @@ func (s *PublicTransactionPoolAPI) GetBlockTransactionCountByHash(ctx context.Co } // GetTransactionByBlockNumberAndIndex returns the transaction for the given block number and index. -func (s *PublicTransactionPoolAPI) GetTransactionByBlockNumberAndIndex(ctx context.Context, blockNr rpc.BlockNumber, index hexutil.Uint) *RPCTransaction { +func (s *TransactionAPI) GetTransactionByBlockNumberAndIndex(ctx context.Context, blockNr rpc.BlockNumber, index hexutil.Uint) *RPCTransaction { if block, _ := s.b.BlockByNumber(ctx, blockNr); block != nil { return newRPCTransactionFromBlockIndex(block, uint64(index), s.b.ChainConfig()) } @@ -1460,7 +1491,7 @@ func (s *PublicTransactionPoolAPI) GetTransactionByBlockNumberAndIndex(ctx conte } // GetTransactionByBlockHashAndIndex returns the transaction for the given block hash and index. -func (s *PublicTransactionPoolAPI) GetTransactionByBlockHashAndIndex(ctx context.Context, blockHash common.Hash, index hexutil.Uint) *RPCTransaction { +func (s *TransactionAPI) GetTransactionByBlockHashAndIndex(ctx context.Context, blockHash common.Hash, index hexutil.Uint) *RPCTransaction { if block, _ := s.b.BlockByHash(ctx, blockHash); block != nil { return newRPCTransactionFromBlockIndex(block, uint64(index), s.b.ChainConfig()) } @@ -1468,7 +1499,7 @@ func (s *PublicTransactionPoolAPI) GetTransactionByBlockHashAndIndex(ctx context } // GetRawTransactionByBlockNumberAndIndex returns the bytes of the transaction for the given block number and index. -func (s *PublicTransactionPoolAPI) GetRawTransactionByBlockNumberAndIndex(ctx context.Context, blockNr rpc.BlockNumber, index hexutil.Uint) hexutil.Bytes { +func (s *TransactionAPI) GetRawTransactionByBlockNumberAndIndex(ctx context.Context, blockNr rpc.BlockNumber, index hexutil.Uint) hexutil.Bytes { if block, _ := s.b.BlockByNumber(ctx, blockNr); block != nil { return newRPCRawTransactionFromBlockIndex(block, uint64(index)) } @@ -1476,7 +1507,7 @@ func (s *PublicTransactionPoolAPI) GetRawTransactionByBlockNumberAndIndex(ctx co } // GetRawTransactionByBlockHashAndIndex returns the bytes of the transaction for the given block hash and index. -func (s *PublicTransactionPoolAPI) GetRawTransactionByBlockHashAndIndex(ctx context.Context, blockHash common.Hash, index hexutil.Uint) hexutil.Bytes { +func (s *TransactionAPI) GetRawTransactionByBlockHashAndIndex(ctx context.Context, blockHash common.Hash, index hexutil.Uint) hexutil.Bytes { if block, _ := s.b.BlockByHash(ctx, blockHash); block != nil { return newRPCRawTransactionFromBlockIndex(block, uint64(index)) } @@ -1484,7 +1515,7 @@ func (s *PublicTransactionPoolAPI) GetRawTransactionByBlockHashAndIndex(ctx cont } // GetTransactionCount returns the number of transactions the given address has sent for the given block number -func (s *PublicTransactionPoolAPI) GetTransactionCount(ctx context.Context, address common.Address, blockNrOrHash rpc.BlockNumberOrHash) (*hexutil.Uint64, error) { +func (s *TransactionAPI) GetTransactionCount(ctx context.Context, address common.Address, blockNrOrHash rpc.BlockNumberOrHash) (*hexutil.Uint64, error) { // Ask transaction pool for the nonce which includes pending transactions if blockNr, ok := blockNrOrHash.Number(); ok && blockNr == rpc.PendingBlockNumber { nonce, err := s.b.GetPoolNonce(ctx, address) @@ -1503,7 +1534,7 @@ func (s *PublicTransactionPoolAPI) GetTransactionCount(ctx context.Context, addr } // GetTransactionByHash returns the transaction for the given hash -func (s *PublicTransactionPoolAPI) GetTransactionByHash(ctx context.Context, hash common.Hash) (*RPCTransaction, error) { +func (s *TransactionAPI) GetTransactionByHash(ctx context.Context, hash common.Hash) (*RPCTransaction, error) { // Try to return an already finalized transaction tx, blockHash, blockNumber, index, err := s.b.GetTransaction(ctx, hash) if err != nil { @@ -1526,7 +1557,7 @@ func (s *PublicTransactionPoolAPI) GetTransactionByHash(ctx context.Context, has } // GetRawTransactionByHash returns the bytes of the transaction for the given hash. -func (s *PublicTransactionPoolAPI) GetRawTransactionByHash(ctx context.Context, hash common.Hash) (hexutil.Bytes, error) { +func (s *TransactionAPI) GetRawTransactionByHash(ctx context.Context, hash common.Hash) (hexutil.Bytes, error) { // Retrieve a finalized transaction, or a pooled otherwise tx, _, _, _, err := s.b.GetTransaction(ctx, hash) if err != nil { @@ -1543,9 +1574,11 @@ func (s *PublicTransactionPoolAPI) GetRawTransactionByHash(ctx context.Context, } // GetTransactionReceipt returns the transaction receipt for the given transaction hash. -func (s *PublicTransactionPoolAPI) GetTransactionReceipt(ctx context.Context, hash common.Hash) (map[string]interface{}, error) { +func (s *TransactionAPI) GetTransactionReceipt(ctx context.Context, hash common.Hash) (map[string]interface{}, error) { tx, blockHash, blockNumber, index, err := s.b.GetTransaction(ctx, hash) if err != nil { + // When the transaction doesn't exist, the RPC method should return JSON null + // as per specification. return nil, nil } receipts, err := s.b.GetReceipts(ctx, blockHash) @@ -1604,7 +1637,7 @@ func (s *PublicTransactionPoolAPI) GetTransactionReceipt(ctx context.Context, ha } // sign is a helper function that signs a transaction with the private key of the given address. -func (s *PublicTransactionPoolAPI) sign(addr common.Address, tx *types.Transaction) (*types.Transaction, error) { +func (s *TransactionAPI) sign(addr common.Address, tx *types.Transaction) (*types.Transaction, error) { // Look up the wallet containing the requested signer account := accounts.Account{Address: addr} @@ -1648,7 +1681,7 @@ func SubmitTransaction(ctx context.Context, b Backend, tx *types.Transaction) (c // SendTransaction creates a transaction for the given argument, sign it and submit it to the // transaction pool. -func (s *PublicTransactionPoolAPI) SendTransaction(ctx context.Context, args TransactionArgs) (common.Hash, error) { +func (s *TransactionAPI) SendTransaction(ctx context.Context, args TransactionArgs) (common.Hash, error) { // Look up the wallet containing the requested signer account := accounts.Account{Address: args.from()} @@ -1681,7 +1714,7 @@ func (s *PublicTransactionPoolAPI) SendTransaction(ctx context.Context, args Tra // FillTransaction fills the defaults (nonce, gas, gasPrice or 1559 fields) // on a given unsigned transaction, and returns it to the caller for further // processing (signing + broadcast). -func (s *PublicTransactionPoolAPI) FillTransaction(ctx context.Context, args TransactionArgs) (*SignTransactionResult, error) { +func (s *TransactionAPI) FillTransaction(ctx context.Context, args TransactionArgs) (*SignTransactionResult, error) { // Set some sanity defaults and terminate on failure if err := args.setDefaults(ctx, s.b); err != nil { return nil, err @@ -1697,7 +1730,7 @@ func (s *PublicTransactionPoolAPI) FillTransaction(ctx context.Context, args Tra // SendRawTransaction will add the signed transaction to the transaction pool. // The sender is responsible for signing the transaction and using the correct nonce. -func (s *PublicTransactionPoolAPI) SendRawTransaction(ctx context.Context, input hexutil.Bytes) (common.Hash, error) { +func (s *TransactionAPI) SendRawTransaction(ctx context.Context, input hexutil.Bytes) (common.Hash, error) { tx := new(types.Transaction) if err := tx.UnmarshalBinary(input); err != nil { return common.Hash{}, err @@ -1706,7 +1739,7 @@ func (s *PublicTransactionPoolAPI) SendRawTransaction(ctx context.Context, input } // Sign calculates an ECDSA signature for: -// keccack256("\x19Ethereum Signed Message:\n" + len(message) + message). +// keccak256("\x19Ethereum Signed Message:\n" + len(message) + message). // // Note, the produced signature conforms to the secp256k1 curve R, S and V values, // where the V value will be 27 or 28 for legacy reasons. @@ -1714,7 +1747,7 @@ func (s *PublicTransactionPoolAPI) SendRawTransaction(ctx context.Context, input // The account associated with addr must be unlocked. // // https://github.com/ethereum/wiki/wiki/JSON-RPC#eth_sign -func (s *PublicTransactionPoolAPI) Sign(addr common.Address, data hexutil.Bytes) (hexutil.Bytes, error) { +func (s *TransactionAPI) Sign(addr common.Address, data hexutil.Bytes) (hexutil.Bytes, error) { // Look up the wallet containing the requested signer account := accounts.Account{Address: addr} @@ -1739,7 +1772,7 @@ type SignTransactionResult struct { // SignTransaction will sign the given transaction with the from account. // The node needs to have the private key of the account corresponding with // the given from address and it needs to be unlocked. -func (s *PublicTransactionPoolAPI) SignTransaction(ctx context.Context, args TransactionArgs) (*SignTransactionResult, error) { +func (s *TransactionAPI) SignTransaction(ctx context.Context, args TransactionArgs) (*SignTransactionResult, error) { if args.Gas == nil { return nil, fmt.Errorf("gas not specified") } @@ -1770,7 +1803,7 @@ func (s *PublicTransactionPoolAPI) SignTransaction(ctx context.Context, args Tra // PendingTransactions returns the transactions that are in the transaction pool // and have a from address that is one of the accounts this node manages. -func (s *PublicTransactionPoolAPI) PendingTransactions() ([]*RPCTransaction, error) { +func (s *TransactionAPI) PendingTransactions() ([]*RPCTransaction, error) { pending, err := s.b.GetPoolTransactions() if err != nil { return nil, err @@ -1794,7 +1827,7 @@ func (s *PublicTransactionPoolAPI) PendingTransactions() ([]*RPCTransaction, err // Resend accepts an existing transaction and a new gas price and limit. It will remove // the given transaction from the pool and reinsert it with the new gas price and limit. -func (s *PublicTransactionPoolAPI) Resend(ctx context.Context, sendArgs TransactionArgs, gasPrice *hexutil.Big, gasLimit *hexutil.Uint64) (common.Hash, error) { +func (s *TransactionAPI) Resend(ctx context.Context, sendArgs TransactionArgs, gasPrice *hexutil.Big, gasLimit *hexutil.Uint64) (common.Hash, error) { if sendArgs.Nonce == nil { return common.Hash{}, fmt.Errorf("missing transaction nonce in transaction spec") } @@ -1844,20 +1877,19 @@ func (s *PublicTransactionPoolAPI) Resend(ctx context.Context, sendArgs Transact return common.Hash{}, fmt.Errorf("transaction %#x not found", matchTx.Hash()) } -// PublicDebugAPI is the collection of Ethereum APIs exposed over the public -// debugging endpoint. -type PublicDebugAPI struct { +// DebugAPI is the collection of Ethereum APIs exposed over the debugging +// namespace. +type DebugAPI struct { b Backend } -// NewPublicDebugAPI creates a new API definition for the public debug methods -// of the Ethereum service. -func NewPublicDebugAPI(b Backend) *PublicDebugAPI { - return &PublicDebugAPI{b: b} +// NewDebugAPI creates a new instance of DebugAPI. +func NewDebugAPI(b Backend) *DebugAPI { + return &DebugAPI{b: b} } // GetHeaderRlp retrieves the RLP encoded for of a single header. -func (api *PublicDebugAPI) GetHeaderRlp(ctx context.Context, number uint64) (hexutil.Bytes, error) { +func (api *DebugAPI) GetHeaderRlp(ctx context.Context, number uint64) (hexutil.Bytes, error) { header, _ := api.b.HeaderByNumber(ctx, rpc.BlockNumber(number)) if header == nil { return nil, fmt.Errorf("header #%d not found", number) @@ -1866,7 +1898,7 @@ func (api *PublicDebugAPI) GetHeaderRlp(ctx context.Context, number uint64) (hex } // GetBlockRlp retrieves the RLP encoded for of a single block. -func (api *PublicDebugAPI) GetBlockRlp(ctx context.Context, number uint64) (hexutil.Bytes, error) { +func (api *DebugAPI) GetBlockRlp(ctx context.Context, number uint64) (hexutil.Bytes, error) { block, _ := api.b.BlockByNumber(ctx, rpc.BlockNumber(number)) if block == nil { return nil, fmt.Errorf("block #%d not found", number) @@ -1875,7 +1907,7 @@ func (api *PublicDebugAPI) GetBlockRlp(ctx context.Context, number uint64) (hexu } // GetRawReceipts retrieves the binary-encoded raw receipts of a single block. -func (api *PublicDebugAPI) GetRawReceipts(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) ([]hexutil.Bytes, error) { +func (api *DebugAPI) GetRawReceipts(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) ([]hexutil.Bytes, error) { var hash common.Hash if h, ok := blockNrOrHash.Hash(); ok { hash = h @@ -1902,7 +1934,7 @@ func (api *PublicDebugAPI) GetRawReceipts(ctx context.Context, blockNrOrHash rpc } // PrintBlock retrieves a block and returns its pretty printed form. -func (api *PublicDebugAPI) PrintBlock(ctx context.Context, number uint64) (string, error) { +func (api *DebugAPI) PrintBlock(ctx context.Context, number uint64) (string, error) { block, _ := api.b.BlockByNumber(ctx, rpc.BlockNumber(number)) if block == nil { return "", fmt.Errorf("block #%d not found", number) @@ -1911,28 +1943,16 @@ func (api *PublicDebugAPI) PrintBlock(ctx context.Context, number uint64) (strin } // SeedHash retrieves the seed hash of a block. -func (api *PublicDebugAPI) SeedHash(ctx context.Context, number uint64) (string, error) { +func (api *DebugAPI) SeedHash(ctx context.Context, number uint64) (string, error) { block, _ := api.b.BlockByNumber(ctx, rpc.BlockNumber(number)) if block == nil { return "", fmt.Errorf("block #%d not found", number) } - return fmt.Sprintf("0x%x", ethash.SeedHash(number)), nil -} - -// PrivateDebugAPI is the collection of Ethereum APIs exposed over the private -// debugging endpoint. -type PrivateDebugAPI struct { - b Backend -} - -// NewPrivateDebugAPI creates a new API definition for the private debug methods -// of the Ethereum service. -func NewPrivateDebugAPI(b Backend) *PrivateDebugAPI { - return &PrivateDebugAPI{b: b} + return fmt.Sprintf("%#x", ethash.SeedHash(number)), nil } // ChaindbProperty returns leveldb properties of the key-value database. -func (api *PrivateDebugAPI) ChaindbProperty(property string) (string, error) { +func (api *DebugAPI) ChaindbProperty(property string) (string, error) { if property == "" { property = "leveldb.stats" } else if !strings.HasPrefix(property, "leveldb.") { @@ -1943,7 +1963,7 @@ func (api *PrivateDebugAPI) ChaindbProperty(property string) (string, error) { // ChaindbCompact flattens the entire key-value database into a single level, // removing all unused slots and merging all keys. -func (api *PrivateDebugAPI) ChaindbCompact() error { +func (api *DebugAPI) ChaindbCompact() error { for b := byte(0); b < 255; b++ { log.Info("Compacting chain database", "range", fmt.Sprintf("0x%0.2X-0x%0.2X", b, b+1)) if err := api.b.ChainDb().Compact([]byte{b}, []byte{b + 1}); err != nil { @@ -1955,33 +1975,33 @@ func (api *PrivateDebugAPI) ChaindbCompact() error { } // SetHead rewinds the head of the blockchain to a previous block. -func (api *PrivateDebugAPI) SetHead(number hexutil.Uint64) { +func (api *DebugAPI) SetHead(number hexutil.Uint64) { api.b.SetHead(uint64(number)) } -// PublicNetAPI offers network related RPC methods -type PublicNetAPI struct { +// NetAPI offers network related RPC methods +type NetAPI struct { net *p2p.Server networkVersion uint64 } -// NewPublicNetAPI creates a new net API instance. -func NewPublicNetAPI(net *p2p.Server, networkVersion uint64) *PublicNetAPI { - return &PublicNetAPI{net, networkVersion} +// NewNetAPI creates a new net API instance. +func NewNetAPI(net *p2p.Server, networkVersion uint64) *NetAPI { + return &NetAPI{net, networkVersion} } // Listening returns an indication if the node is listening for network connections. -func (s *PublicNetAPI) Listening() bool { +func (s *NetAPI) Listening() bool { return true // always listening } // PeerCount returns the number of connected peers -func (s *PublicNetAPI) PeerCount() hexutil.Uint { +func (s *NetAPI) PeerCount() hexutil.Uint { return hexutil.Uint(s.net.PeerCount()) } // Version returns the current ethereum protocol version. -func (s *PublicNetAPI) Version() string { +func (s *NetAPI) Version() string { return fmt.Sprintf("%d", s.networkVersion) } diff --git a/internal/ethapi/backend.go b/internal/ethapi/backend.go index bc60fb2a64f6c..5b4ceb6310691 100644 --- a/internal/ethapi/backend.go +++ b/internal/ethapi/backend.go @@ -27,10 +27,10 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/bloombits" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/params" @@ -65,6 +65,7 @@ type Backend interface { BlockByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Block, error) StateAndHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*state.StateDB, *types.Header, error) StateAndHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*state.StateDB, *types.Header, error) + PendingBlockAndReceipts() (*types.Block, types.Receipts) GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) GetTd(ctx context.Context, hash common.Hash) *big.Int GetEVM(ctx context.Context, msg core.Message, state *state.StateDB, header *types.Header, vmConfig *vm.Config) (*vm.EVM, func() error, error) @@ -83,16 +84,12 @@ type Backend interface { TxPoolContentFrom(addr common.Address) (types.Transactions, types.Transactions) SubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscription - // Filter API - BloomStatus() (uint64, uint64) - GetLogs(ctx context.Context, blockHash common.Hash) ([][]*types.Log, error) - ServiceFilter(ctx context.Context, session *bloombits.MatcherSession) - SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription - SubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription - SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription - ChainConfig() *params.ChainConfig Engine() consensus.Engine + + // eth/filters needs to be initialized from this backend type, so methods needed by + // it must also be included here. + filters.Backend } func GetAPIs(apiBackend Backend) []rpc.API { @@ -100,43 +97,25 @@ func GetAPIs(apiBackend Backend) []rpc.API { return []rpc.API{ { Namespace: "eth", - Version: "1.0", - Service: NewPublicEthereumAPI(apiBackend), - Public: true, + Service: NewEthereumAPI(apiBackend), }, { Namespace: "eth", - Version: "1.0", - Service: NewPublicBlockChainAPI(apiBackend), - Public: true, + Service: NewBlockChainAPI(apiBackend), }, { Namespace: "eth", - Version: "1.0", - Service: NewPublicTransactionPoolAPI(apiBackend, nonceLock), - Public: true, + Service: NewTransactionAPI(apiBackend, nonceLock), }, { Namespace: "txpool", - Version: "1.0", - Service: NewPublicTxPoolAPI(apiBackend), - Public: true, - }, { - Namespace: "debug", - Version: "1.0", - Service: NewPublicDebugAPI(apiBackend), - Public: true, + Service: NewTxPoolAPI(apiBackend), }, { Namespace: "debug", - Version: "1.0", - Service: NewPrivateDebugAPI(apiBackend), + Service: NewDebugAPI(apiBackend), }, { Namespace: "eth", - Version: "1.0", - Service: NewPublicAccountAPI(apiBackend.AccountManager()), - Public: true, + Service: NewEthereumAccountAPI(apiBackend.AccountManager()), }, { Namespace: "personal", - Version: "1.0", - Service: NewPrivateAccountAPI(apiBackend, nonceLock), - Public: false, + Service: NewPersonalAccountAPI(apiBackend, nonceLock), }, } } diff --git a/internal/ethapi/dbapi.go b/internal/ethapi/dbapi.go index 33dca29d3c66f..33fda936dcd0f 100644 --- a/internal/ethapi/dbapi.go +++ b/internal/ethapi/dbapi.go @@ -22,7 +22,7 @@ import ( ) // DbGet returns the raw value of a key stored in the database. -func (api *PrivateDebugAPI) DbGet(key string) (hexutil.Bytes, error) { +func (api *DebugAPI) DbGet(key string) (hexutil.Bytes, error) { blob, err := common.ParseHexOrString(key) if err != nil { return nil, err @@ -32,12 +32,12 @@ func (api *PrivateDebugAPI) DbGet(key string) (hexutil.Bytes, error) { // DbAncient retrieves an ancient binary blob from the append-only immutable files. // It is a mapping to the `AncientReaderOp.Ancient` method -func (api *PrivateDebugAPI) DbAncient(kind string, number uint64) (hexutil.Bytes, error) { +func (api *DebugAPI) DbAncient(kind string, number uint64) (hexutil.Bytes, error) { return api.b.ChainDb().Ancient(kind, number) } // DbAncients returns the ancient item numbers in the ancient store. // It is a mapping to the `AncientReaderOp.Ancients` method -func (api *PrivateDebugAPI) DbAncients() (uint64, error) { +func (api *DebugAPI) DbAncients() (uint64, error) { return api.b.ChainDb().Ancients() } diff --git a/internal/ethapi/transaction_args.go b/internal/ethapi/transaction_args.go index 9c5950af58fe6..e07248db5d693 100644 --- a/internal/ethapi/transaction_args.go +++ b/internal/ethapi/transaction_args.go @@ -75,56 +75,8 @@ func (args *TransactionArgs) data() []byte { // setDefaults fills in default values for unspecified tx fields. func (args *TransactionArgs) setDefaults(ctx context.Context, b Backend) error { - if args.GasPrice != nil && (args.MaxFeePerGas != nil || args.MaxPriorityFeePerGas != nil) { - return errors.New("both gasPrice and (maxFeePerGas or maxPriorityFeePerGas) specified") - } - // After london, default to 1559 unless gasPrice is set - head := b.CurrentHeader() - // If user specifies both maxPriorityfee and maxFee, then we do not - // need to consult the chain for defaults. It's definitely a London tx. - if args.MaxPriorityFeePerGas == nil || args.MaxFeePerGas == nil { - // In this clause, user left some fields unspecified. - if b.ChainConfig().IsLondon(head.Number) && args.GasPrice == nil { - if args.MaxPriorityFeePerGas == nil { - tip, err := b.SuggestGasTipCap(ctx) - if err != nil { - return err - } - args.MaxPriorityFeePerGas = (*hexutil.Big)(tip) - } - if args.MaxFeePerGas == nil { - gasFeeCap := new(big.Int).Add( - (*big.Int)(args.MaxPriorityFeePerGas), - new(big.Int).Mul(head.BaseFee, big.NewInt(2)), - ) - args.MaxFeePerGas = (*hexutil.Big)(gasFeeCap) - } - if args.MaxFeePerGas.ToInt().Cmp(args.MaxPriorityFeePerGas.ToInt()) < 0 { - return fmt.Errorf("maxFeePerGas (%v) < maxPriorityFeePerGas (%v)", args.MaxFeePerGas, args.MaxPriorityFeePerGas) - } - } else { - if args.MaxFeePerGas != nil || args.MaxPriorityFeePerGas != nil { - return errors.New("maxFeePerGas or maxPriorityFeePerGas specified but london is not active yet") - } - if args.GasPrice == nil { - price, err := b.SuggestGasTipCap(ctx) - if err != nil { - return err - } - if b.ChainConfig().IsLondon(head.Number) { - // The legacy tx gas price suggestion should not add 2x base fee - // because all fees are consumed, so it would result in a spiral - // upwards. - price.Add(price, head.BaseFee) - } - args.GasPrice = (*hexutil.Big)(price) - } - } - } else { - // Both maxPriorityfee and maxFee set by caller. Sanity-check their internal relation - if args.MaxFeePerGas.ToInt().Cmp(args.MaxPriorityFeePerGas.ToInt()) < 0 { - return fmt.Errorf("maxFeePerGas (%v) < maxPriorityFeePerGas (%v)", args.MaxFeePerGas, args.MaxPriorityFeePerGas) - } + if err := args.setFeeDefaults(ctx, b); err != nil { + return err } if args.Value == nil { args.Value = new(hexutil.Big) @@ -165,9 +117,81 @@ func (args *TransactionArgs) setDefaults(ctx context.Context, b Backend) error { args.Gas = &estimated log.Trace("Estimate gas usage automatically", "gas", args.Gas) } - if args.ChainID == nil { - id := (*hexutil.Big)(b.ChainConfig().ChainID) - args.ChainID = id + // If chain id is provided, ensure it matches the local chain id. Otherwise, set the local + // chain id as the default. + want := b.ChainConfig().ChainID + if args.ChainID != nil { + if have := (*big.Int)(args.ChainID); have.Cmp(want) != 0 { + return fmt.Errorf("chainId does not match node's (have=%v, want=%v)", have, want) + } + } else { + args.ChainID = (*hexutil.Big)(want) + } + return nil +} + +// setFeeDefaults fills in default fee values for unspecified tx fields. +func (args *TransactionArgs) setFeeDefaults(ctx context.Context, b Backend) error { + // If both gasPrice and at least one of the EIP-1559 fee parameters are specified, error. + if args.GasPrice != nil && (args.MaxFeePerGas != nil || args.MaxPriorityFeePerGas != nil) { + return errors.New("both gasPrice and (maxFeePerGas or maxPriorityFeePerGas) specified") + } + // If the tx has completely specified a fee mechanism, no default is needed. This allows users + // who are not yet synced past London to get defaults for other tx values. See + // https://github.com/ethereum/go-ethereum/pull/23274 for more information. + eip1559ParamsSet := args.MaxFeePerGas != nil && args.MaxPriorityFeePerGas != nil + if (args.GasPrice != nil && !eip1559ParamsSet) || (args.GasPrice == nil && eip1559ParamsSet) { + // Sanity check the EIP-1559 fee parameters if present. + if args.GasPrice == nil && args.MaxFeePerGas.ToInt().Cmp(args.MaxPriorityFeePerGas.ToInt()) < 0 { + return fmt.Errorf("maxFeePerGas (%v) < maxPriorityFeePerGas (%v)", args.MaxFeePerGas, args.MaxPriorityFeePerGas) + } + return nil + } + // Now attempt to fill in default value depending on whether London is active or not. + head := b.CurrentHeader() + if b.ChainConfig().IsLondon(head.Number) { + // London is active, set maxPriorityFeePerGas and maxFeePerGas. + if err := args.setLondonFeeDefaults(ctx, head, b); err != nil { + return err + } + } else { + if args.MaxFeePerGas != nil || args.MaxPriorityFeePerGas != nil { + return fmt.Errorf("maxFeePerGas and maxPriorityFeePerGas are not valid before London is active") + } + // London not active, set gas price. + price, err := b.SuggestGasTipCap(ctx) + if err != nil { + return err + } + args.GasPrice = (*hexutil.Big)(price) + } + return nil +} + +// setLondonFeeDefaults fills in reasonable default fee values for unspecified fields. +func (args *TransactionArgs) setLondonFeeDefaults(ctx context.Context, head *types.Header, b Backend) error { + // Set maxPriorityFeePerGas if it is missing. + if args.MaxPriorityFeePerGas == nil { + tip, err := b.SuggestGasTipCap(ctx) + if err != nil { + return err + } + args.MaxPriorityFeePerGas = (*hexutil.Big)(tip) + } + // Set maxFeePerGas if it is missing. + if args.MaxFeePerGas == nil { + // Set the max fee to be 2 times larger than the previous block's base fee. + // The additional slack allows the tx to not become invalidated if the base + // fee is rising. + val := new(big.Int).Add( + args.MaxPriorityFeePerGas.ToInt(), + new(big.Int).Mul(head.BaseFee, big.NewInt(2)), + ) + args.MaxFeePerGas = (*hexutil.Big)(val) + } + // Both EIP-1559 fee parameters are now set; sanity check them. + if args.MaxFeePerGas.ToInt().Cmp(args.MaxPriorityFeePerGas.ToInt()) < 0 { + return fmt.Errorf("maxFeePerGas (%v) < maxPriorityFeePerGas (%v)", args.MaxFeePerGas, args.MaxPriorityFeePerGas) } return nil } @@ -214,7 +238,7 @@ func (args *TransactionArgs) ToMessage(globalGasCap uint64, baseFee *big.Int) (t gasPrice = args.GasPrice.ToInt() gasFeeCap, gasTipCap = gasPrice, gasPrice } else { - // User specified 1559 gas feilds (or none), use those + // User specified 1559 gas fields (or none), use those gasFeeCap = new(big.Int) if args.MaxFeePerGas != nil { gasFeeCap = args.MaxFeePerGas.ToInt() diff --git a/internal/ethapi/transaction_args_test.go b/internal/ethapi/transaction_args_test.go new file mode 100644 index 0000000000000..28dc561c36e4c --- /dev/null +++ b/internal/ethapi/transaction_args_test.go @@ -0,0 +1,342 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package ethapi + +import ( + "context" + "fmt" + "math/big" + "reflect" + "testing" + "time" + + "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/accounts" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/consensus" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/bloombits" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/event" + "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/rpc" +) + +// TestSetFeeDefaults tests the logic for filling in default fee values works as expected. +func TestSetFeeDefaults(t *testing.T) { + type test struct { + name string + isLondon bool + in *TransactionArgs + want *TransactionArgs + err error + } + + var ( + b = newBackendMock() + fortytwo = (*hexutil.Big)(big.NewInt(42)) + maxFee = (*hexutil.Big)(new(big.Int).Add(new(big.Int).Mul(b.current.BaseFee, big.NewInt(2)), fortytwo.ToInt())) + al = &types.AccessList{types.AccessTuple{Address: common.Address{0xaa}, StorageKeys: []common.Hash{{0x01}}}} + ) + + tests := []test{ + // Legacy txs + { + "legacy tx pre-London", + false, + &TransactionArgs{}, + &TransactionArgs{GasPrice: fortytwo}, + nil, + }, + { + "legacy tx post-London, explicit gas price", + true, + &TransactionArgs{GasPrice: fortytwo}, + &TransactionArgs{GasPrice: fortytwo}, + nil, + }, + + // Access list txs + { + "access list tx pre-London", + false, + &TransactionArgs{AccessList: al}, + &TransactionArgs{AccessList: al, GasPrice: fortytwo}, + nil, + }, + { + "access list tx post-London, explicit gas price", + false, + &TransactionArgs{AccessList: al, GasPrice: fortytwo}, + &TransactionArgs{AccessList: al, GasPrice: fortytwo}, + nil, + }, + { + "access list tx post-London", + true, + &TransactionArgs{AccessList: al}, + &TransactionArgs{AccessList: al, MaxFeePerGas: maxFee, MaxPriorityFeePerGas: fortytwo}, + nil, + }, + { + "access list tx post-London, only max fee", + true, + &TransactionArgs{AccessList: al, MaxFeePerGas: maxFee}, + &TransactionArgs{AccessList: al, MaxFeePerGas: maxFee, MaxPriorityFeePerGas: fortytwo}, + nil, + }, + { + "access list tx post-London, only priority fee", + true, + &TransactionArgs{AccessList: al, MaxFeePerGas: maxFee}, + &TransactionArgs{AccessList: al, MaxFeePerGas: maxFee, MaxPriorityFeePerGas: fortytwo}, + nil, + }, + + // Dynamic fee txs + { + "dynamic tx post-London", + true, + &TransactionArgs{}, + &TransactionArgs{MaxFeePerGas: maxFee, MaxPriorityFeePerGas: fortytwo}, + nil, + }, + { + "dynamic tx post-London, only max fee", + true, + &TransactionArgs{MaxFeePerGas: maxFee}, + &TransactionArgs{MaxFeePerGas: maxFee, MaxPriorityFeePerGas: fortytwo}, + nil, + }, + { + "dynamic tx post-London, only priority fee", + true, + &TransactionArgs{MaxFeePerGas: maxFee}, + &TransactionArgs{MaxFeePerGas: maxFee, MaxPriorityFeePerGas: fortytwo}, + nil, + }, + { + "dynamic fee tx pre-London, maxFee set", + false, + &TransactionArgs{MaxFeePerGas: maxFee}, + nil, + fmt.Errorf("maxFeePerGas and maxPriorityFeePerGas are not valid before London is active"), + }, + { + "dynamic fee tx pre-London, priorityFee set", + false, + &TransactionArgs{MaxPriorityFeePerGas: fortytwo}, + nil, + fmt.Errorf("maxFeePerGas and maxPriorityFeePerGas are not valid before London is active"), + }, + { + "dynamic fee tx, maxFee < priorityFee", + true, + &TransactionArgs{MaxFeePerGas: maxFee, MaxPriorityFeePerGas: (*hexutil.Big)(big.NewInt(1000))}, + nil, + fmt.Errorf("maxFeePerGas (0x3e) < maxPriorityFeePerGas (0x3e8)"), + }, + { + "dynamic fee tx, maxFee < priorityFee while setting default", + true, + &TransactionArgs{MaxFeePerGas: (*hexutil.Big)(big.NewInt(7))}, + nil, + fmt.Errorf("maxFeePerGas (0x7) < maxPriorityFeePerGas (0x2a)"), + }, + + // Misc + { + "set all fee parameters", + false, + &TransactionArgs{GasPrice: fortytwo, MaxFeePerGas: maxFee, MaxPriorityFeePerGas: fortytwo}, + nil, + fmt.Errorf("both gasPrice and (maxFeePerGas or maxPriorityFeePerGas) specified"), + }, + { + "set gas price and maxPriorityFee", + false, + &TransactionArgs{GasPrice: fortytwo, MaxPriorityFeePerGas: fortytwo}, + nil, + fmt.Errorf("both gasPrice and (maxFeePerGas or maxPriorityFeePerGas) specified"), + }, + { + "set gas price and maxFee", + true, + &TransactionArgs{GasPrice: fortytwo, MaxFeePerGas: maxFee}, + nil, + fmt.Errorf("both gasPrice and (maxFeePerGas or maxPriorityFeePerGas) specified"), + }, + } + + ctx := context.Background() + for i, test := range tests { + if test.isLondon { + b.activateLondon() + } else { + b.deactivateLondon() + } + got := test.in + err := got.setFeeDefaults(ctx, b) + if err != nil && err.Error() == test.err.Error() { + // Test threw expected error. + continue + } else if err != nil { + t.Fatalf("test %d (%s): unexpected error: %s", i, test.name, err) + } + if !reflect.DeepEqual(got, test.want) { + t.Fatalf("test %d (%s): did not fill defaults as expected: (got: %v, want: %v)", i, test.name, got, test.want) + } + } +} + +type backendMock struct { + current *types.Header + config *params.ChainConfig +} + +func newBackendMock() *backendMock { + config := ¶ms.ChainConfig{ + ChainID: big.NewInt(42), + HomesteadBlock: big.NewInt(0), + DAOForkBlock: nil, + DAOForkSupport: true, + EIP150Block: big.NewInt(0), + EIP155Block: big.NewInt(0), + EIP158Block: big.NewInt(0), + ByzantiumBlock: big.NewInt(0), + ConstantinopleBlock: big.NewInt(0), + PetersburgBlock: big.NewInt(0), + IstanbulBlock: big.NewInt(0), + MuirGlacierBlock: big.NewInt(0), + BerlinBlock: big.NewInt(0), + LondonBlock: big.NewInt(1000), + } + return &backendMock{ + current: &types.Header{ + Difficulty: big.NewInt(10000000000), + Number: big.NewInt(1100), + GasLimit: 8_000_000, + GasUsed: 8_000_000, + Time: 555, + Extra: make([]byte, 32), + BaseFee: big.NewInt(10), + }, + config: config, + } +} + +func (b *backendMock) activateLondon() { + b.current.Number = big.NewInt(1100) +} + +func (b *backendMock) deactivateLondon() { + b.current.Number = big.NewInt(900) +} +func (b *backendMock) SuggestGasTipCap(ctx context.Context) (*big.Int, error) { + return big.NewInt(42), nil +} +func (b *backendMock) CurrentHeader() *types.Header { return b.current } +func (b *backendMock) ChainConfig() *params.ChainConfig { return b.config } + +// Other methods needed to implement Backend interface. +func (b *backendMock) SyncProgress() ethereum.SyncProgress { return ethereum.SyncProgress{} } +func (b *backendMock) FeeHistory(ctx context.Context, blockCount int, lastBlock rpc.BlockNumber, rewardPercentiles []float64) (*big.Int, [][]*big.Int, []*big.Int, []float64, error) { + return nil, nil, nil, nil, nil +} +func (b *backendMock) ChainDb() ethdb.Database { return nil } +func (b *backendMock) AccountManager() *accounts.Manager { return nil } +func (b *backendMock) ExtRPCEnabled() bool { return false } +func (b *backendMock) RPCGasCap() uint64 { return 0 } +func (b *backendMock) RPCEVMTimeout() time.Duration { return time.Second } +func (b *backendMock) RPCTxFeeCap() float64 { return 0 } +func (b *backendMock) UnprotectedAllowed() bool { return false } +func (b *backendMock) SetHead(number uint64) {} +func (b *backendMock) HeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Header, error) { + return nil, nil +} +func (b *backendMock) HeaderByHash(ctx context.Context, hash common.Hash) (*types.Header, error) { + return nil, nil +} +func (b *backendMock) HeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Header, error) { + return nil, nil +} +func (b *backendMock) CurrentBlock() *types.Block { return nil } +func (b *backendMock) BlockByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Block, error) { + return nil, nil +} +func (b *backendMock) BlockByHash(ctx context.Context, hash common.Hash) (*types.Block, error) { + return nil, nil +} +func (b *backendMock) BlockByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Block, error) { + return nil, nil +} +func (b *backendMock) StateAndHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*state.StateDB, *types.Header, error) { + return nil, nil, nil +} +func (b *backendMock) StateAndHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*state.StateDB, *types.Header, error) { + return nil, nil, nil +} +func (b *backendMock) PendingBlockAndReceipts() (*types.Block, types.Receipts) { return nil, nil } +func (b *backendMock) GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) { + return nil, nil +} +func (b *backendMock) GetLogs(ctx context.Context, blockHash common.Hash, number uint64) ([][]*types.Log, error) { + return nil, nil +} +func (b *backendMock) GetTd(ctx context.Context, hash common.Hash) *big.Int { return nil } +func (b *backendMock) GetEVM(ctx context.Context, msg core.Message, state *state.StateDB, header *types.Header, vmConfig *vm.Config) (*vm.EVM, func() error, error) { + return nil, nil, nil +} +func (b *backendMock) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription { return nil } +func (b *backendMock) SubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) event.Subscription { + return nil +} +func (b *backendMock) SubscribeChainSideEvent(ch chan<- core.ChainSideEvent) event.Subscription { + return nil +} +func (b *backendMock) SendTx(ctx context.Context, signedTx *types.Transaction) error { return nil } +func (b *backendMock) GetTransaction(ctx context.Context, txHash common.Hash) (*types.Transaction, common.Hash, uint64, uint64, error) { + return nil, [32]byte{}, 0, 0, nil +} +func (b *backendMock) GetPoolTransactions() (types.Transactions, error) { return nil, nil } +func (b *backendMock) GetPoolTransaction(txHash common.Hash) *types.Transaction { return nil } +func (b *backendMock) GetPoolNonce(ctx context.Context, addr common.Address) (uint64, error) { + return 0, nil +} +func (b *backendMock) Stats() (pending int, queued int) { return 0, 0 } +func (b *backendMock) TxPoolContent() (map[common.Address]types.Transactions, map[common.Address]types.Transactions) { + return nil, nil +} +func (b *backendMock) TxPoolContentFrom(addr common.Address) (types.Transactions, types.Transactions) { + return nil, nil +} +func (b *backendMock) SubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscription { return nil } +func (b *backendMock) BloomStatus() (uint64, uint64) { return 0, 0 } +func (b *backendMock) ServiceFilter(ctx context.Context, session *bloombits.MatcherSession) {} +func (b *backendMock) SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription { return nil } +func (b *backendMock) SubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription { + return nil +} +func (b *backendMock) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription { + return nil +} + +func (b *backendMock) Engine() consensus.Engine { return nil } diff --git a/internal/flags/categories.go b/internal/flags/categories.go new file mode 100644 index 0000000000000..c2db6c6c1d25c --- /dev/null +++ b/internal/flags/categories.go @@ -0,0 +1,43 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package flags + +import "github.com/urfave/cli/v2" + +const ( + EthCategory = "ETHEREUM" + LightCategory = "LIGHT CLIENT" + DevCategory = "DEVELOPER CHAIN" + EthashCategory = "ETHASH" + TxPoolCategory = "TRANSACTION POOL" + PerfCategory = "PERFORMANCE TUNING" + AccountCategory = "ACCOUNT" + APICategory = "API AND CONSOLE" + NetworkingCategory = "NETWORKING" + MinerCategory = "MINER" + GasPriceCategory = "GAS PRICE ORACLE" + VMCategory = "VIRTUAL MACHINE" + LoggingCategory = "LOGGING AND DEBUGGING" + MetricsCategory = "METRICS AND STATS" + MiscCategory = "MISC" + DeprecatedCategory = "ALIASED (deprecated)" +) + +func init() { + cli.HelpFlag.(*cli.BoolFlag).Category = MiscCategory + cli.VersionFlag.(*cli.BoolFlag).Category = MiscCategory +} diff --git a/internal/flags/flags.go b/internal/flags/flags.go new file mode 100644 index 0000000000000..0ae2c6a512efe --- /dev/null +++ b/internal/flags/flags.go @@ -0,0 +1,340 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package flags + +import ( + "encoding" + "errors" + "flag" + "math/big" + "os" + "os/user" + "path" + "strings" + + "github.com/ethereum/go-ethereum/common/math" + "github.com/urfave/cli/v2" +) + +// DirectoryString is custom type which is registered in the flags library which cli uses for +// argument parsing. This allows us to expand Value to an absolute path when +// the argument is parsed +type DirectoryString string + +func (s *DirectoryString) String() string { + return string(*s) +} + +func (s *DirectoryString) Set(value string) error { + *s = DirectoryString(expandPath(value)) + return nil +} + +var ( + _ cli.Flag = (*DirectoryFlag)(nil) + _ cli.RequiredFlag = (*DirectoryFlag)(nil) + _ cli.VisibleFlag = (*DirectoryFlag)(nil) + _ cli.DocGenerationFlag = (*DirectoryFlag)(nil) + _ cli.CategorizableFlag = (*DirectoryFlag)(nil) +) + +// DirectoryFlag is custom cli.Flag type which expand the received string to an absolute path. +// e.g. ~/.ethereum -> /home/username/.ethereum +type DirectoryFlag struct { + Name string + + Category string + DefaultText string + Usage string + + Required bool + Hidden bool + HasBeenSet bool + + Value DirectoryString + + Aliases []string +} + +// For cli.Flag: + +func (f *DirectoryFlag) Names() []string { return append([]string{f.Name}, f.Aliases...) } +func (f *DirectoryFlag) IsSet() bool { return f.HasBeenSet } +func (f *DirectoryFlag) String() string { return cli.FlagStringer(f) } + +// Apply called by cli library, grabs variable from environment (if in env) +// and adds variable to flag set for parsing. +func (f *DirectoryFlag) Apply(set *flag.FlagSet) error { + eachName(f, func(name string) { + set.Var(&f.Value, f.Name, f.Usage) + }) + return nil +} + +// For cli.RequiredFlag: + +func (f *DirectoryFlag) IsRequired() bool { return f.Required } + +// For cli.VisibleFlag: + +func (f *DirectoryFlag) IsVisible() bool { return !f.Hidden } + +// For cli.CategorizableFlag: + +func (f *DirectoryFlag) GetCategory() string { return f.Category } + +// For cli.DocGenerationFlag: + +func (f *DirectoryFlag) TakesValue() bool { return true } +func (f *DirectoryFlag) GetUsage() string { return f.Usage } +func (f *DirectoryFlag) GetValue() string { return f.Value.String() } +func (f *DirectoryFlag) GetEnvVars() []string { return nil } // env not supported + +func (f *DirectoryFlag) GetDefaultText() string { + if f.DefaultText != "" { + return f.DefaultText + } + return f.GetValue() +} + +type TextMarshaler interface { + encoding.TextMarshaler + encoding.TextUnmarshaler +} + +// textMarshalerVal turns a TextMarshaler into a flag.Value +type textMarshalerVal struct { + v TextMarshaler +} + +func (v textMarshalerVal) String() string { + if v.v == nil { + return "" + } + text, _ := v.v.MarshalText() + return string(text) +} + +func (v textMarshalerVal) Set(s string) error { + return v.v.UnmarshalText([]byte(s)) +} + +var ( + _ cli.Flag = (*TextMarshalerFlag)(nil) + _ cli.RequiredFlag = (*TextMarshalerFlag)(nil) + _ cli.VisibleFlag = (*TextMarshalerFlag)(nil) + _ cli.DocGenerationFlag = (*TextMarshalerFlag)(nil) + _ cli.CategorizableFlag = (*TextMarshalerFlag)(nil) +) + +// TextMarshalerFlag wraps a TextMarshaler value. +type TextMarshalerFlag struct { + Name string + + Category string + DefaultText string + Usage string + + Required bool + Hidden bool + HasBeenSet bool + + Value TextMarshaler + + Aliases []string +} + +// For cli.Flag: + +func (f *TextMarshalerFlag) Names() []string { return append([]string{f.Name}, f.Aliases...) } +func (f *TextMarshalerFlag) IsSet() bool { return f.HasBeenSet } +func (f *TextMarshalerFlag) String() string { return cli.FlagStringer(f) } + +func (f *TextMarshalerFlag) Apply(set *flag.FlagSet) error { + eachName(f, func(name string) { + set.Var(textMarshalerVal{f.Value}, f.Name, f.Usage) + }) + return nil +} + +// For cli.RequiredFlag: + +func (f *TextMarshalerFlag) IsRequired() bool { return f.Required } + +// For cli.VisibleFlag: + +func (f *TextMarshalerFlag) IsVisible() bool { return !f.Hidden } + +// For cli.CategorizableFlag: + +func (f *TextMarshalerFlag) GetCategory() string { return f.Category } + +// For cli.DocGenerationFlag: + +func (f *TextMarshalerFlag) TakesValue() bool { return true } +func (f *TextMarshalerFlag) GetUsage() string { return f.Usage } +func (f *TextMarshalerFlag) GetEnvVars() []string { return nil } // env not supported + +func (f *TextMarshalerFlag) GetValue() string { + t, err := f.Value.MarshalText() + if err != nil { + return "(ERR: " + err.Error() + ")" + } + return string(t) +} + +func (f *TextMarshalerFlag) GetDefaultText() string { + if f.DefaultText != "" { + return f.DefaultText + } + return f.GetValue() +} + +// GlobalTextMarshaler returns the value of a TextMarshalerFlag from the global flag set. +func GlobalTextMarshaler(ctx *cli.Context, name string) TextMarshaler { + val := ctx.Generic(name) + if val == nil { + return nil + } + return val.(textMarshalerVal).v +} + +var ( + _ cli.Flag = (*BigFlag)(nil) + _ cli.RequiredFlag = (*BigFlag)(nil) + _ cli.VisibleFlag = (*BigFlag)(nil) + _ cli.DocGenerationFlag = (*BigFlag)(nil) + _ cli.CategorizableFlag = (*BigFlag)(nil) +) + +// BigFlag is a command line flag that accepts 256 bit big integers in decimal or +// hexadecimal syntax. +type BigFlag struct { + Name string + + Category string + DefaultText string + Usage string + + Required bool + Hidden bool + HasBeenSet bool + + Value *big.Int + + Aliases []string +} + +// For cli.Flag: + +func (f *BigFlag) Names() []string { return append([]string{f.Name}, f.Aliases...) } +func (f *BigFlag) IsSet() bool { return f.HasBeenSet } +func (f *BigFlag) String() string { return cli.FlagStringer(f) } + +func (f *BigFlag) Apply(set *flag.FlagSet) error { + eachName(f, func(name string) { + f.Value = new(big.Int) + set.Var((*bigValue)(f.Value), f.Name, f.Usage) + }) + + return nil +} + +// For cli.RequiredFlag: + +func (f *BigFlag) IsRequired() bool { return f.Required } + +// For cli.VisibleFlag: + +func (f *BigFlag) IsVisible() bool { return !f.Hidden } + +// For cli.CategorizableFlag: + +func (f *BigFlag) GetCategory() string { return f.Category } + +// For cli.DocGenerationFlag: + +func (f *BigFlag) TakesValue() bool { return true } +func (f *BigFlag) GetUsage() string { return f.Usage } +func (f *BigFlag) GetValue() string { return f.Value.String() } +func (f *BigFlag) GetEnvVars() []string { return nil } // env not supported + +func (f *BigFlag) GetDefaultText() string { + if f.DefaultText != "" { + return f.DefaultText + } + return f.GetValue() +} + +// bigValue turns *big.Int into a flag.Value +type bigValue big.Int + +func (b *bigValue) String() string { + if b == nil { + return "" + } + return (*big.Int)(b).String() +} + +func (b *bigValue) Set(s string) error { + intVal, ok := math.ParseBig256(s) + if !ok { + return errors.New("invalid integer syntax") + } + *b = (bigValue)(*intVal) + return nil +} + +// GlobalBig returns the value of a BigFlag from the global flag set. +func GlobalBig(ctx *cli.Context, name string) *big.Int { + val := ctx.Generic(name) + if val == nil { + return nil + } + return (*big.Int)(val.(*bigValue)) +} + +// Expands a file path +// 1. replace tilde with users home dir +// 2. expands embedded environment variables +// 3. cleans the path, e.g. /a/b/../c -> /a/c +// Note, it has limitations, e.g. ~someuser/tmp will not be expanded +func expandPath(p string) string { + if strings.HasPrefix(p, "~/") || strings.HasPrefix(p, "~\\") { + if home := HomeDir(); home != "" { + p = home + p[1:] + } + } + return path.Clean(os.ExpandEnv(p)) +} + +func HomeDir() string { + if home := os.Getenv("HOME"); home != "" { + return home + } + if usr, err := user.Current(); err == nil { + return usr.HomeDir + } + return "" +} + +func eachName(f cli.Flag, fn func(string)) { + for _, name := range f.Names() { + name = strings.Trim(name, " ") + fn(name) + } +} diff --git a/cmd/utils/customflags_test.go b/internal/flags/flags_test.go similarity index 61% rename from cmd/utils/customflags_test.go rename to internal/flags/flags_test.go index de39ca36a116c..a0d4af7ca3606 100644 --- a/cmd/utils/customflags_test.go +++ b/internal/flags/flags_test.go @@ -1,20 +1,20 @@ // Copyright 2015 The go-ethereum Authors -// This file is part of go-ethereum. +// This file is part of the go-ethereum library. // -// go-ethereum is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // -// go-ethereum is distributed in the hope that it will be useful, +// The go-ethereum library is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. +// GNU Lesser General Public License for more details. // -// You should have received a copy of the GNU General Public License -// along with go-ethereum. If not, see . +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . -package utils +package flags import ( "os" diff --git a/internal/flags/helpers.go b/internal/flags/helpers.go index 1fc6409c6550d..4bcdc816fe463 100644 --- a/internal/flags/helpers.go +++ b/internal/flags/helpers.go @@ -17,137 +17,173 @@ package flags import ( - "os" - "path/filepath" + "fmt" + "strings" "github.com/ethereum/go-ethereum/params" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) -var ( - CommandHelpTemplate = `{{.cmd.Name}}{{if .cmd.Subcommands}} command{{end}}{{if .cmd.Flags}} [command options]{{end}} {{.cmd.ArgsUsage}} -{{if .cmd.Description}}{{.cmd.Description}} -{{end}}{{if .cmd.Subcommands}} -SUBCOMMANDS: - {{range .cmd.Subcommands}}{{.Name}}{{with .ShortName}}, {{.}}{{end}}{{ "\t" }}{{.Usage}} - {{end}}{{end}}{{if .categorizedFlags}} -{{range $idx, $categorized := .categorizedFlags}}{{$categorized.Name}} OPTIONS: -{{range $categorized.Flags}}{{"\t"}}{{.}} -{{end}} -{{end}}{{end}}` - - OriginCommandHelpTemplate = `{{.Name}}{{if .Subcommands}} command{{end}}{{if .Flags}} [command options]{{end}} {{.ArgsUsage}} -{{if .Description}}{{.Description}} -{{end}}{{if .Subcommands}} -SUBCOMMANDS: - {{range .Subcommands}}{{.Name}}{{with .ShortName}}, {{.}}{{end}}{{ "\t" }}{{.Usage}} - {{end}}{{end}}{{if .Flags}} -OPTIONS: -{{range $.Flags}} {{.}} -{{end}} -{{end}}` - - // AppHelpTemplate is the test template for the default, global app help topic. - AppHelpTemplate = `NAME: - {{.App.Name}} - {{.App.Usage}} - - Copyright 2013-2022 The go-ethereum Authors - -USAGE: - {{.App.HelpName}} [options]{{if .App.Commands}} [command] [command options]{{end}} {{if .App.ArgsUsage}}{{.App.ArgsUsage}}{{else}}[arguments...]{{end}} - {{if .App.Version}} -VERSION: - {{.App.Version}} - {{end}}{{if len .App.Authors}} -AUTHOR(S): - {{range .App.Authors}}{{ . }}{{end}} - {{end}}{{if .App.Commands}} -COMMANDS: - {{range .App.Commands}}{{join .Names ", "}}{{ "\t" }}{{.Usage}} - {{end}}{{end}}{{if .FlagGroups}} -{{range .FlagGroups}}{{.Name}} OPTIONS: - {{range .Flags}}{{.}} - {{end}} -{{end}}{{end}}{{if .App.Copyright }} -COPYRIGHT: - {{.App.Copyright}} - {{end}} -` - // ClefAppHelpTemplate is the template for the default, global app help topic. - ClefAppHelpTemplate = `NAME: - {{.App.Name}} - {{.App.Usage}} - - Copyright 2013-2022 The go-ethereum Authors - -USAGE: - {{.App.HelpName}} [options]{{if .App.Commands}} command [command options]{{end}} {{if .App.ArgsUsage}}{{.App.ArgsUsage}}{{else}}[arguments...]{{end}} - {{if .App.Version}} -COMMANDS: - {{range .App.Commands}}{{join .Names ", "}}{{ "\t" }}{{.Usage}} - {{end}}{{end}}{{if .FlagGroups}} -{{range .FlagGroups}}{{.Name}} OPTIONS: - {{range .Flags}}{{.}} - {{end}} -{{end}}{{end}}{{if .App.Copyright }} -COPYRIGHT: - {{.App.Copyright}} - {{end}} -` -) - -// HelpData is a one shot struct to pass to the usage template -type HelpData struct { - App interface{} - FlagGroups []FlagGroup +// NewApp creates an app with sane defaults. +func NewApp(gitCommit, gitDate, usage string) *cli.App { + app := cli.NewApp() + app.EnableBashCompletion = true + app.Version = params.VersionWithCommit(gitCommit, gitDate) + app.Usage = usage + app.Copyright = "Copyright 2013-2022 The go-ethereum Authors" + app.Before = func(ctx *cli.Context) error { + MigrateGlobalFlags(ctx) + return nil + } + return app } -// FlagGroup is a collection of flags belonging to a single topic. -type FlagGroup struct { - Name string - Flags []cli.Flag +// Merge merges the given flag slices. +func Merge(groups ...[]cli.Flag) []cli.Flag { + var ret []cli.Flag + for _, group := range groups { + ret = append(ret, group...) + } + return ret } -// ByCategory sorts an array of FlagGroup by Name in the order -// defined in AppHelpFlagGroups. -type ByCategory []FlagGroup +var migrationApplied = map[*cli.Command]struct{}{} + +// MigrateGlobalFlags makes all global flag values available in the +// context. This should be called as early as possible in app.Before. +// +// Example: +// +// geth account new --keystore /tmp/mykeystore --lightkdf +// +// is equivalent after calling this method with: +// +// geth --keystore /tmp/mykeystore --lightkdf account new +// +// i.e. in the subcommand Action function of 'account new', ctx.Bool("lightkdf) +// will return true even if --lightkdf is set as a global option. +// +// This function may become unnecessary when https://github.com/urfave/cli/pull/1245 is merged. +func MigrateGlobalFlags(ctx *cli.Context) { + var iterate func(cs []*cli.Command, fn func(*cli.Command)) + iterate = func(cs []*cli.Command, fn func(*cli.Command)) { + for _, cmd := range cs { + if _, ok := migrationApplied[cmd]; ok { + continue + } + migrationApplied[cmd] = struct{}{} + fn(cmd) + iterate(cmd.Subcommands, fn) + } + } -func (a ByCategory) Len() int { return len(a) } -func (a ByCategory) Swap(i, j int) { a[i], a[j] = a[j], a[i] } -func (a ByCategory) Less(i, j int) bool { - iCat, jCat := a[i].Name, a[j].Name - iIdx, jIdx := len(a), len(a) // ensure non categorized flags come last + // This iterates over all commands and wraps their action function. + iterate(ctx.App.Commands, func(cmd *cli.Command) { + if cmd.Action == nil { + return + } - for i, group := range a { - if iCat == group.Name { - iIdx = i + action := cmd.Action + cmd.Action = func(ctx *cli.Context) error { + doMigrateFlags(ctx) + return action(ctx) } - if jCat == group.Name { - jIdx = i + }) +} + +func doMigrateFlags(ctx *cli.Context) { + for _, name := range ctx.FlagNames() { + for _, parent := range ctx.Lineage()[1:] { + if parent.IsSet(name) { + ctx.Set(name, parent.String(name)) + break + } } } +} - return iIdx < jIdx +func init() { + cli.FlagStringer = FlagString } -func FlagCategory(flag cli.Flag, flagGroups []FlagGroup) string { - for _, category := range flagGroups { - for _, flg := range category.Flags { - if flg.GetName() == flag.GetName() { - return category.Name +// FlagString prints a single flag in help. +func FlagString(f cli.Flag) string { + df, ok := f.(cli.DocGenerationFlag) + if !ok { + return "" + } + + needsPlaceholder := df.TakesValue() + placeholder := "" + if needsPlaceholder { + placeholder = "value" + } + + namesText := pad(cli.FlagNamePrefixer(df.Names(), placeholder), 30) + + defaultValueString := "" + if s := df.GetDefaultText(); s != "" { + defaultValueString = " (default: " + s + ")" + } + + usage := strings.TrimSpace(df.GetUsage()) + envHint := strings.TrimSpace(cli.FlagEnvHinter(df.GetEnvVars(), "")) + if len(envHint) > 0 { + usage += " " + envHint + } + + usage = wordWrap(usage, 80) + usage = indent(usage, 10) + + return fmt.Sprintf("\n %s%s\n%s", namesText, defaultValueString, usage) +} + +func pad(s string, length int) string { + if len(s) < length { + s += strings.Repeat(" ", length-len(s)) + } + return s +} + +func indent(s string, nspace int) string { + ind := strings.Repeat(" ", nspace) + return ind + strings.ReplaceAll(s, "\n", "\n"+ind) +} + +func wordWrap(s string, width int) string { + var ( + output strings.Builder + lineLength = 0 + ) + + for { + sp := strings.IndexByte(s, ' ') + var word string + if sp == -1 { + word = s + } else { + word = s[:sp] + } + wlen := len(word) + over := lineLength+wlen >= width + if over { + output.WriteByte('\n') + lineLength = 0 + } else { + if lineLength != 0 { + output.WriteByte(' ') + lineLength++ } } + + output.WriteString(word) + lineLength += wlen + + if sp == -1 { + break + } + s = s[wlen+1:] } - return "MISC" -} -// NewApp creates an app with sane defaults. -func NewApp(gitCommit, gitDate, usage string) *cli.App { - app := cli.NewApp() - app.EnableBashCompletion = true - app.Name = filepath.Base(os.Args[0]) - app.Author = "" - app.Email = "" - app.Version = params.VersionWithCommit(gitCommit, gitDate) - app.Usage = usage - return app + return output.String() } diff --git a/internal/jsre/deps/web3.js b/internal/jsre/deps/web3.js index f82d93bdc570f..a291218ec51f9 100644 --- a/internal/jsre/deps/web3.js +++ b/internal/jsre/deps/web3.js @@ -3696,7 +3696,7 @@ var outputBigNumberFormatter = function (number) { }; var isPredefinedBlockNumber = function (blockNumber) { - return blockNumber === 'latest' || blockNumber === 'pending' || blockNumber === 'earliest' || blockNumber === 'finalized'; + return blockNumber === 'latest' || blockNumber === 'pending' || blockNumber === 'earliest' || blockNumber === 'finalized' || blockNumber === 'safe'; }; var inputDefaultBlockNumberFormatter = function (blockNumber) { diff --git a/internal/jsre/jsre.go b/internal/jsre/jsre.go index 4de80a9e901c0..f6e21d2ef7016 100644 --- a/internal/jsre/jsre.go +++ b/internal/jsre/jsre.go @@ -322,11 +322,11 @@ func (re *JSRE) loadScript(call Call) (goja.Value, error) { file = common.AbsolutePath(re.assetPath, file) source, err := os.ReadFile(file) if err != nil { - return nil, fmt.Errorf("Could not read file %s: %v", file, err) + return nil, fmt.Errorf("could not read file %s: %v", file, err) } value, err := compileAndRun(re.vm, file, string(source)) if err != nil { - return nil, fmt.Errorf("Error while compiling or running script: %v", err) + return nil, fmt.Errorf("error while compiling or running script: %v", err) } return value, nil } diff --git a/internal/jsre/pretty.go b/internal/jsre/pretty.go index 4171e00906173..bd772b4927c2d 100644 --- a/internal/jsre/pretty.go +++ b/internal/jsre/pretty.go @@ -219,7 +219,6 @@ func (ctx ppctx) fields(obj *goja.Object) []string { vals = append(vals, k) } } - } iterOwnAndConstructorKeys(ctx.vm, obj, add) sort.Strings(vals) diff --git a/les/api.go b/les/api.go index 782bb31ef29a0..dc86393817775 100644 --- a/les/api.go +++ b/les/api.go @@ -33,15 +33,15 @@ var ( errUnknownBenchmarkType = errors.New("unknown benchmark type") ) -// PrivateLightServerAPI provides an API to access the LES light server. -type PrivateLightServerAPI struct { +// LightServerAPI provides an API to access the LES light server. +type LightServerAPI struct { server *LesServer defaultPosFactors, defaultNegFactors vfs.PriceFactors } -// NewPrivateLightServerAPI creates a new LES light server API. -func NewPrivateLightServerAPI(server *LesServer) *PrivateLightServerAPI { - return &PrivateLightServerAPI{ +// NewLightServerAPI creates a new LES light server API. +func NewLightServerAPI(server *LesServer) *LightServerAPI { + return &LightServerAPI{ server: server, defaultPosFactors: defaultPosFactors, defaultNegFactors: defaultNegFactors, @@ -61,7 +61,7 @@ func parseNode(node string) (enode.ID, error) { } // ServerInfo returns global server parameters -func (api *PrivateLightServerAPI) ServerInfo() map[string]interface{} { +func (api *LightServerAPI) ServerInfo() map[string]interface{} { res := make(map[string]interface{}) res["minimumCapacity"] = api.server.minCapacity res["maximumCapacity"] = api.server.maxCapacity @@ -72,7 +72,7 @@ func (api *PrivateLightServerAPI) ServerInfo() map[string]interface{} { } // ClientInfo returns information about clients listed in the ids list or matching the given tags -func (api *PrivateLightServerAPI) ClientInfo(nodes []string) map[enode.ID]map[string]interface{} { +func (api *LightServerAPI) ClientInfo(nodes []string) map[enode.ID]map[string]interface{} { var ids []enode.ID for _, node := range nodes { if id, err := parseNode(node); err == nil { @@ -102,7 +102,7 @@ func (api *PrivateLightServerAPI) ClientInfo(nodes []string) map[enode.ID]map[st // If maxCount limit is applied but there are more potential results then the ID // of the next potential result is included in the map with an empty structure // assigned to it. -func (api *PrivateLightServerAPI) PriorityClientInfo(start, stop enode.ID, maxCount int) map[enode.ID]map[string]interface{} { +func (api *LightServerAPI) PriorityClientInfo(start, stop enode.ID, maxCount int) map[enode.ID]map[string]interface{} { res := make(map[enode.ID]map[string]interface{}) ids := api.server.clientPool.GetPosBalanceIDs(start, stop, maxCount+1) if len(ids) > maxCount { @@ -122,7 +122,7 @@ func (api *PrivateLightServerAPI) PriorityClientInfo(start, stop enode.ID, maxCo } // clientInfo creates a client info data structure -func (api *PrivateLightServerAPI) clientInfo(peer *clientPeer, balance vfs.ReadOnlyBalance) map[string]interface{} { +func (api *LightServerAPI) clientInfo(peer *clientPeer, balance vfs.ReadOnlyBalance) map[string]interface{} { info := make(map[string]interface{}) pb, nb := balance.GetBalance() info["isConnected"] = peer != nil @@ -140,7 +140,7 @@ func (api *PrivateLightServerAPI) clientInfo(peer *clientPeer, balance vfs.ReadO // setParams either sets the given parameters for a single connected client (if specified) // or the default parameters applicable to clients connected in the future -func (api *PrivateLightServerAPI) setParams(params map[string]interface{}, client *clientPeer, posFactors, negFactors *vfs.PriceFactors) (updateFactors bool, err error) { +func (api *LightServerAPI) setParams(params map[string]interface{}, client *clientPeer, posFactors, negFactors *vfs.PriceFactors) (updateFactors bool, err error) { defParams := client == nil for name, value := range params { errValue := func() error { @@ -191,7 +191,7 @@ func (api *PrivateLightServerAPI) setParams(params map[string]interface{}, clien // SetClientParams sets client parameters for all clients listed in the ids list // or all connected clients if the list is empty -func (api *PrivateLightServerAPI) SetClientParams(nodes []string, params map[string]interface{}) error { +func (api *LightServerAPI) SetClientParams(nodes []string, params map[string]interface{}) error { var err error for _, node := range nodes { var id enode.ID @@ -215,7 +215,7 @@ func (api *PrivateLightServerAPI) SetClientParams(nodes []string, params map[str } // SetDefaultParams sets the default parameters applicable to clients connected in the future -func (api *PrivateLightServerAPI) SetDefaultParams(params map[string]interface{}) error { +func (api *LightServerAPI) SetDefaultParams(params map[string]interface{}) error { update, err := api.setParams(params, nil, &api.defaultPosFactors, &api.defaultNegFactors) if update { api.server.clientPool.SetDefaultFactors(api.defaultPosFactors, api.defaultNegFactors) @@ -227,7 +227,7 @@ func (api *PrivateLightServerAPI) SetDefaultParams(params map[string]interface{} // So that already connected client won't be kicked out very soon and we can ensure all // connected clients can have enough time to request or sync some data. // When the input parameter `bias` < 0 (illegal), return error. -func (api *PrivateLightServerAPI) SetConnectedBias(bias time.Duration) error { +func (api *LightServerAPI) SetConnectedBias(bias time.Duration) error { if bias < time.Duration(0) { return fmt.Errorf("bias illegal: %v less than 0", bias) } @@ -237,7 +237,7 @@ func (api *PrivateLightServerAPI) SetConnectedBias(bias time.Duration) error { // AddBalance adds the given amount to the balance of a client if possible and returns // the balance before and after the operation -func (api *PrivateLightServerAPI) AddBalance(node string, amount int64) (balance [2]uint64, err error) { +func (api *LightServerAPI) AddBalance(node string, amount int64) (balance [2]uint64, err error) { var id enode.ID if id, err = parseNode(node); err != nil { return @@ -254,7 +254,7 @@ func (api *PrivateLightServerAPI) AddBalance(node string, amount int64) (balance // // Note: measurement time is adjusted for each pass depending on the previous ones. // Therefore a controlled total measurement time is achievable in multiple passes. -func (api *PrivateLightServerAPI) Benchmark(setups []map[string]interface{}, passCount, length int) ([]map[string]interface{}, error) { +func (api *LightServerAPI) Benchmark(setups []map[string]interface{}, passCount, length int) ([]map[string]interface{}, error) { benchmarks := make([]requestBenchmark, len(setups)) for i, setup := range setups { if t, ok := setup["type"].(string); ok { @@ -324,20 +324,20 @@ func (api *PrivateLightServerAPI) Benchmark(setups []map[string]interface{}, pas return result, nil } -// PrivateDebugAPI provides an API to debug LES light server functionality. -type PrivateDebugAPI struct { +// DebugAPI provides an API to debug LES light server functionality. +type DebugAPI struct { server *LesServer } -// NewPrivateDebugAPI creates a new LES light server debug API. -func NewPrivateDebugAPI(server *LesServer) *PrivateDebugAPI { - return &PrivateDebugAPI{ +// NewDebugAPI creates a new LES light server debug API. +func NewDebugAPI(server *LesServer) *DebugAPI { + return &DebugAPI{ server: server, } } // FreezeClient forces a temporary client freeze which normally happens when the server is overloaded -func (api *PrivateDebugAPI) FreezeClient(node string) error { +func (api *DebugAPI) FreezeClient(node string) error { var ( id enode.ID err error @@ -353,14 +353,14 @@ func (api *PrivateDebugAPI) FreezeClient(node string) error { } } -// PrivateLightAPI provides an API to access the LES light server or light client. -type PrivateLightAPI struct { +// LightAPI provides an API to access the LES light server or light client. +type LightAPI struct { backend *lesCommons } -// NewPrivateLightAPI creates a new LES service API. -func NewPrivateLightAPI(backend *lesCommons) *PrivateLightAPI { - return &PrivateLightAPI{backend: backend} +// NewLightAPI creates a new LES service API. +func NewLightAPI(backend *lesCommons) *LightAPI { + return &LightAPI{backend: backend} } // LatestCheckpoint returns the latest local checkpoint package. @@ -370,7 +370,7 @@ func NewPrivateLightAPI(backend *lesCommons) *PrivateLightAPI { // result[1], 32 bytes hex encoded latest section head hash // result[2], 32 bytes hex encoded latest section canonical hash trie root hash // result[3], 32 bytes hex encoded latest section bloom trie root hash -func (api *PrivateLightAPI) LatestCheckpoint() ([4]string, error) { +func (api *LightAPI) LatestCheckpoint() ([4]string, error) { var res [4]string cp := api.backend.latestLocalCheckpoint() if cp.Empty() { @@ -387,7 +387,7 @@ func (api *PrivateLightAPI) LatestCheckpoint() ([4]string, error) { // result[0], 32 bytes hex encoded latest section head hash // result[1], 32 bytes hex encoded latest section canonical hash trie root hash // result[2], 32 bytes hex encoded latest section bloom trie root hash -func (api *PrivateLightAPI) GetCheckpoint(index uint64) ([3]string, error) { +func (api *LightAPI) GetCheckpoint(index uint64) ([3]string, error) { var res [3]string cp := api.backend.localCheckpoint(index) if cp.Empty() { @@ -398,7 +398,7 @@ func (api *PrivateLightAPI) GetCheckpoint(index uint64) ([3]string, error) { } // GetCheckpointContractAddress returns the contract contract address in hex format. -func (api *PrivateLightAPI) GetCheckpointContractAddress() (string, error) { +func (api *LightAPI) GetCheckpointContractAddress() (string, error) { if api.backend.oracle == nil { return "", errNotActivated } diff --git a/les/api_backend.go b/les/api_backend.go index 11a9ca128aab5..5b4213134b248 100644 --- a/les/api_backend.go +++ b/les/api_backend.go @@ -168,11 +168,8 @@ func (b *LesApiBackend) GetReceipts(ctx context.Context, hash common.Hash) (type return nil, nil } -func (b *LesApiBackend) GetLogs(ctx context.Context, hash common.Hash) ([][]*types.Log, error) { - if number := rawdb.ReadHeaderNumber(b.eth.chainDb, hash); number != nil { - return light.GetBlockLogs(ctx, b.eth.odr, hash, *number) - } - return nil, nil +func (b *LesApiBackend) GetLogs(ctx context.Context, hash common.Hash, number uint64) ([][]*types.Log, error) { + return light.GetBlockLogs(ctx, b.eth.odr, hash, number) } func (b *LesApiBackend) GetTd(ctx context.Context, hash common.Hash) *big.Int { diff --git a/les/api_test.go b/les/api_test.go index ea6870e356275..3db1c5fd5ec9c 100644 --- a/les/api_test.go +++ b/les/api_test.go @@ -340,7 +340,6 @@ func freezeClient(ctx context.Context, t *testing.T, server *rpc.Client, clientI if err := server.CallContext(ctx, nil, "debug_freezeClient", clientID); err != nil { t.Fatalf("Failed to freeze client: %v", err) } - } func setCapacity(ctx context.Context, t *testing.T, server *rpc.Client, clientID enode.ID, cap uint64) { diff --git a/les/catalyst/api.go b/les/catalyst/api.go index de09acdb02133..983fc7bff0bc9 100644 --- a/les/catalyst/api.go +++ b/les/catalyst/api.go @@ -35,9 +35,7 @@ func Register(stack *node.Node, backend *les.LightEthereum) error { stack.RegisterAPIs([]rpc.API{ { Namespace: "engine", - Version: "1.0", Service: NewConsensusAPI(backend), - Public: true, Authenticated: true, }, }) @@ -52,7 +50,7 @@ type ConsensusAPI struct { // The underlying blockchain needs to have a valid terminal total difficulty set. func NewConsensusAPI(les *les.LightEthereum) *ConsensusAPI { if les.BlockChain().Config().TerminalTotalDifficulty == nil { - panic("Catalyst started without valid total difficulty") + log.Warn("Catalyst started without valid total difficulty") } return &ConsensusAPI{les: les} } diff --git a/les/catalyst/api_test.go b/les/catalyst/api_test.go index 70a6d24719ea3..26c49d6ef908b 100644 --- a/les/catalyst/api_test.go +++ b/les/catalyst/api_test.go @@ -55,7 +55,7 @@ func generatePreMergeChain(n int) (*core.Genesis, []*types.Header, []*types.Bloc Timestamp: 9000, BaseFee: big.NewInt(params.InitialBaseFee), } - gblock := genesis.ToBlock(db) + gblock := genesis.MustCommit(db) engine := ethash.NewFaker() blocks, _ := core.GenerateChain(config, gblock, engine, db, n, nil) totalDifficulty := big.NewInt(0) diff --git a/les/client.go b/les/client.go index c3acbc2e4bf8a..6504fe2af8f67 100644 --- a/les/client.go +++ b/les/client.go @@ -19,6 +19,7 @@ package les import ( "fmt" + "strings" "time" "github.com/ethereum/go-ethereum/accounts" @@ -31,7 +32,6 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth/ethconfig" - "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/eth/gasprice" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/internal/ethapi" @@ -73,7 +73,7 @@ type LightEthereum struct { eventMux *event.TypeMux engine consensus.Engine accountManager *accounts.Manager - netRPCService *ethapi.PublicNetAPI + netRPCService *ethapi.NetAPI p2pServer *p2p.Server p2pConfig *p2p.Config @@ -92,11 +92,17 @@ func New(stack *node.Node, config *ethconfig.Config) (*LightEthereum, error) { if err != nil { return nil, err } - chainConfig, genesisHash, genesisErr := core.SetupGenesisBlockWithOverride(chainDb, config.Genesis, config.OverrideArrowGlacier, config.OverrideTerminalTotalDifficulty) + chainConfig, genesisHash, genesisErr := core.SetupGenesisBlockWithOverride(chainDb, config.Genesis, config.OverrideTerminalTotalDifficulty, config.OverrideTerminalTotalDifficultyPassed) if _, isCompat := genesisErr.(*params.ConfigCompatError); genesisErr != nil && !isCompat { return nil, genesisErr } - log.Info("Initialised chain configuration", "config", chainConfig) + log.Info("") + log.Info(strings.Repeat("-", 153)) + for _, line := range strings.Split(chainConfig.String(), "\n") { + log.Info(line) + } + log.Info(strings.Repeat("-", 153)) + log.Info("") peers := newServerPeerSet() merger := consensus.NewMerger(chainDb) @@ -182,7 +188,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*LightEthereum, error) { leth.blockchain.DisableCheckFreq() } - leth.netRPCService = ethapi.NewPublicNetAPI(leth.p2pServer, leth.config.NetworkId) + leth.netRPCService = ethapi.NewNetAPI(leth.p2pServer, leth.config.NetworkId) // Register the backend on the node stack.RegisterAPIs(leth.APIs()) @@ -287,34 +293,19 @@ func (s *LightEthereum) APIs() []rpc.API { return append(apis, []rpc.API{ { Namespace: "eth", - Version: "1.0", Service: &LightDummyAPI{}, - Public: true, - }, { - Namespace: "eth", - Version: "1.0", - Service: downloader.NewPublicDownloaderAPI(s.handler.downloader, s.eventMux), - Public: true, }, { Namespace: "eth", - Version: "1.0", - Service: filters.NewPublicFilterAPI(s.ApiBackend, true, 5*time.Minute), - Public: true, + Service: downloader.NewDownloaderAPI(s.handler.downloader, s.eventMux), }, { Namespace: "net", - Version: "1.0", Service: s.netRPCService, - Public: true, }, { Namespace: "les", - Version: "1.0", - Service: NewPrivateLightAPI(&s.lesCommons), - Public: false, + Service: NewLightAPI(&s.lesCommons), }, { Namespace: "vflux", - Version: "1.0", Service: s.serverPool.API(), - Public: false, }, }...) } diff --git a/les/distributor.go b/les/distributor.go index 31150e4d731a7..a0319c67f7373 100644 --- a/les/distributor.go +++ b/les/distributor.go @@ -256,7 +256,7 @@ func (d *requestDistributor) queue(r *distReq) chan distPeer { if r.reqOrder == 0 { d.lastReqOrder++ r.reqOrder = d.lastReqOrder - r.waitForPeers = d.clock.Now() + mclock.AbsTime(waitForPeers) + r.waitForPeers = d.clock.Now().Add(waitForPeers) } // Assign the timestamp when the request is queued no matter it's // a new one or re-queued one. diff --git a/les/downloader/api.go b/les/downloader/api.go index 2024d23deade6..21200b676c644 100644 --- a/les/downloader/api.go +++ b/les/downloader/api.go @@ -25,21 +25,21 @@ import ( "github.com/ethereum/go-ethereum/rpc" ) -// PublicDownloaderAPI provides an API which gives information about the current synchronisation status. +// DownloaderAPI provides an API which gives information about the current synchronisation status. // It offers only methods that operates on data that can be available to anyone without security risks. -type PublicDownloaderAPI struct { +type DownloaderAPI struct { d *Downloader mux *event.TypeMux installSyncSubscription chan chan interface{} uninstallSyncSubscription chan *uninstallSyncSubscriptionRequest } -// NewPublicDownloaderAPI create a new PublicDownloaderAPI. The API has an internal event loop that +// NewDownloaderAPI create a new PublicDownloaderAPI. The API has an internal event loop that // listens for events from the downloader through the global event mux. In case it receives one of // these events it broadcasts it to all syncing subscriptions that are installed through the // installSyncSubscription channel. -func NewPublicDownloaderAPI(d *Downloader, m *event.TypeMux) *PublicDownloaderAPI { - api := &PublicDownloaderAPI{ +func NewDownloaderAPI(d *Downloader, m *event.TypeMux) *DownloaderAPI { + api := &DownloaderAPI{ d: d, mux: m, installSyncSubscription: make(chan chan interface{}), @@ -53,7 +53,7 @@ func NewPublicDownloaderAPI(d *Downloader, m *event.TypeMux) *PublicDownloaderAP // eventLoop runs a loop until the event mux closes. It will install and uninstall new // sync subscriptions and broadcasts sync status updates to the installed sync subscriptions. -func (api *PublicDownloaderAPI) eventLoop() { +func (api *DownloaderAPI) eventLoop() { var ( sub = api.mux.Subscribe(StartEvent{}, DoneEvent{}, FailedEvent{}) syncSubscriptions = make(map[chan interface{}]struct{}) @@ -90,7 +90,7 @@ func (api *PublicDownloaderAPI) eventLoop() { } // Syncing provides information when this nodes starts synchronising with the Ethereum network and when it's finished. -func (api *PublicDownloaderAPI) Syncing(ctx context.Context) (*rpc.Subscription, error) { +func (api *DownloaderAPI) Syncing(ctx context.Context) (*rpc.Subscription, error) { notifier, supported := rpc.NotifierFromContext(ctx) if !supported { return &rpc.Subscription{}, rpc.ErrNotificationsUnsupported @@ -125,7 +125,7 @@ type SyncingResult struct { Status ethereum.SyncProgress `json:"status"` } -// uninstallSyncSubscriptionRequest uninstalles a syncing subscription in the API event loop. +// uninstallSyncSubscriptionRequest uninstalls a syncing subscription in the API event loop. type uninstallSyncSubscriptionRequest struct { c chan interface{} uninstalled chan interface{} @@ -133,9 +133,9 @@ type uninstallSyncSubscriptionRequest struct { // SyncStatusSubscription represents a syncing subscription. type SyncStatusSubscription struct { - api *PublicDownloaderAPI // register subscription in event loop of this api instance - c chan interface{} // channel where events are broadcasted to - unsubOnce sync.Once // make sure unsubscribe logic is executed once + api *DownloaderAPI // register subscription in event loop of this api instance + c chan interface{} // channel where events are broadcasted to + unsubOnce sync.Once // make sure unsubscribe logic is executed once } // Unsubscribe uninstalls the subscription from the DownloadAPI event loop. @@ -160,7 +160,7 @@ func (s *SyncStatusSubscription) Unsubscribe() { // SubscribeSyncStatus creates a subscription that will broadcast new synchronisation updates. // The given channel must receive interface values, the result can either -func (api *PublicDownloaderAPI) SubscribeSyncStatus(status chan interface{}) *SyncStatusSubscription { +func (api *DownloaderAPI) SubscribeSyncStatus(status chan interface{}) *SyncStatusSubscription { api.installSyncSubscription <- status return &SyncStatusSubscription{api: api, c: status} } diff --git a/les/downloader/downloader.go b/les/downloader/downloader.go index 448a94192b876..6352b1c21729e 100644 --- a/les/downloader/downloader.go +++ b/les/downloader/downloader.go @@ -1330,7 +1330,6 @@ func (d *Downloader) fetchParts(deliveryCh chan dataPack, deliver func(dataPack) expire func() map[string]int, pending func() int, inFlight func() bool, reserve func(*peerConnection, int) (*fetchRequest, bool, bool), fetchHook func([]*types.Header), fetch func(*peerConnection, *fetchRequest) error, cancel func(*fetchRequest), capacity func(*peerConnection) int, idle func() ([]*peerConnection, int), setIdle func(*peerConnection, int, time.Time), kind string) error { - // Create a ticker to detect expired retrieval tasks ticker := time.NewTicker(100 * time.Millisecond) defer ticker.Stop() @@ -1626,7 +1625,7 @@ func (d *Downloader) processHeaders(origin uint64, td *big.Int) error { log.Warn("Invalid header encountered", "number", chunk[n].Number, "hash", chunk[n].Hash(), "parent", chunk[n].ParentHash, "err", err) return fmt.Errorf("%w: %v", errInvalidChain, err) } - // All verifications passed, track all headers within the alloted limits + // All verifications passed, track all headers within the allotted limits if mode == FastSync { head := chunk[len(chunk)-1].Number.Uint64() if head-rollback > uint64(fsHeaderSafetyNet) { @@ -1664,7 +1663,7 @@ func (d *Downloader) processHeaders(origin uint64, td *big.Int) error { } d.syncStatsLock.Unlock() - // Signal the content downloaders of the availablility of new tasks + // Signal the content downloaders of the availability of new tasks for _, ch := range []chan bool{d.bodyWakeCh, d.receiptWakeCh} { select { case ch <- true: diff --git a/les/downloader/downloader_test.go b/les/downloader/downloader_test.go index f6510eb412378..c56870ff17845 100644 --- a/les/downloader/downloader_test.go +++ b/les/downloader/downloader_test.go @@ -229,7 +229,7 @@ func (dl *downloadTester) CurrentFastBlock() *types.Block { func (dl *downloadTester) FastSyncCommitHead(hash common.Hash) error { // For now only check that the state trie is correct if block := dl.GetBlockByHash(hash); block != nil { - _, err := trie.NewSecure(block.Root(), trie.NewDatabase(dl.stateDb)) + _, err := trie.NewStateTrie(common.Hash{}, block.Root(), trie.NewDatabase(dl.stateDb)) return err } return fmt.Errorf("non existent block: %x", hash[:4]) @@ -621,7 +621,6 @@ func testThrottling(t *testing.T, protocol uint, mode SyncMode) { t.Fatalf("block synchronization failed: %v", err) } tester.terminate() - } // Tests that simple synchronization against a forked chain works correctly. In @@ -654,8 +653,8 @@ func testForkedSync(t *testing.T, protocol uint, mode SyncMode) { assertOwnForkedChain(t, tester, testChainBase.len(), []int{chainA.len(), chainB.len()}) } -// Tests that synchronising against a much shorter but much heavyer fork works -// corrently and is not dropped. +// Tests that synchronising against a much shorter but much heavier fork works +// correctly and is not dropped. func TestHeavyForkedSync66Full(t *testing.T) { testHeavyForkedSync(t, eth.ETH66, FullSync) } func TestHeavyForkedSync66Fast(t *testing.T) { testHeavyForkedSync(t, eth.ETH66, FastSync) } func TestHeavyForkedSync66Light(t *testing.T) { testHeavyForkedSync(t, eth.ETH66, LightSync) } diff --git a/les/downloader/peer.go b/les/downloader/peer.go index 8632948329711..c2161e2dae425 100644 --- a/les/downloader/peer.go +++ b/les/downloader/peer.go @@ -350,6 +350,7 @@ func (ps *peerSet) Register(p *peerConnection) error { } p.rates = msgrate.NewTracker(ps.rates.MeanCapacities(), ps.rates.MedianRoundTrip()) if err := ps.rates.Track(p.id, p.rates); err != nil { + ps.lock.Unlock() return err } ps.peers[p.id] = p @@ -413,7 +414,7 @@ func (ps *peerSet) HeaderIdlePeers() ([]*peerConnection, int) { throughput := func(p *peerConnection) int { return p.rates.Capacity(eth.BlockHeadersMsg, time.Second) } - return ps.idlePeers(eth.ETH66, eth.ETH66, idle, throughput) + return ps.idlePeers(eth.ETH66, eth.ETH67, idle, throughput) } // BodyIdlePeers retrieves a flat list of all the currently body-idle peers within @@ -425,7 +426,7 @@ func (ps *peerSet) BodyIdlePeers() ([]*peerConnection, int) { throughput := func(p *peerConnection) int { return p.rates.Capacity(eth.BlockBodiesMsg, time.Second) } - return ps.idlePeers(eth.ETH66, eth.ETH66, idle, throughput) + return ps.idlePeers(eth.ETH66, eth.ETH67, idle, throughput) } // ReceiptIdlePeers retrieves a flat list of all the currently receipt-idle peers @@ -437,7 +438,7 @@ func (ps *peerSet) ReceiptIdlePeers() ([]*peerConnection, int) { throughput := func(p *peerConnection) int { return p.rates.Capacity(eth.ReceiptsMsg, time.Second) } - return ps.idlePeers(eth.ETH66, eth.ETH66, idle, throughput) + return ps.idlePeers(eth.ETH66, eth.ETH67, idle, throughput) } // NodeDataIdlePeers retrieves a flat list of all the currently node-data-idle @@ -449,7 +450,7 @@ func (ps *peerSet) NodeDataIdlePeers() ([]*peerConnection, int) { throughput := func(p *peerConnection) int { return p.rates.Capacity(eth.NodeDataMsg, time.Second) } - return ps.idlePeers(eth.ETH66, eth.ETH66, idle, throughput) + return ps.idlePeers(eth.ETH66, eth.ETH67, idle, throughput) } // idlePeers retrieves a flat list of all currently idle peers satisfying the diff --git a/les/downloader/queue.go b/les/downloader/queue.go index 04ec12cfa9e7c..b165b6b5c1c78 100644 --- a/les/downloader/queue.go +++ b/les/downloader/queue.go @@ -833,7 +833,6 @@ func (q *queue) deliver(id string, taskPool map[common.Hash]*types.Header, taskQueue *prque.Prque, pendPool map[string]*fetchRequest, reqTimer metrics.Timer, results int, validate func(index int, header *types.Header) error, reconstruct func(index int, result *fetchResult)) (int, error) { - // Short circuit if the data was never requested request := pendPool[id] if request == nil { @@ -873,7 +872,7 @@ func (q *queue) deliver(id string, taskPool map[common.Hash]*types.Header, if res, stale, err := q.resultCache.GetDeliverySlot(header.Number.Uint64()); err == nil { reconstruct(accepted, res) } else { - // else: betweeen here and above, some other peer filled this result, + // else: between here and above, some other peer filled this result, // or it was indeed a no-op. This should not happen, but if it does it's // not something to panic about log.Error("Delivery stale", "stale", stale, "number", header.Number.Uint64(), "err", err) diff --git a/les/downloader/queue_test.go b/les/downloader/queue_test.go index 2a884d30aabac..44b2208595ff4 100644 --- a/les/downloader/queue_test.go +++ b/les/downloader/queue_test.go @@ -27,23 +27,17 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" ) -var ( - testdb = rawdb.NewMemoryDatabase() - genesis = core.GenesisBlockForTesting(testdb, testAddress, big.NewInt(1000000000000000)) -) - // makeChain creates a chain of n blocks starting at and including parent. // the returned hash chain is ordered head->parent. In addition, every 3rd block // contains a transaction and every 5th an uncle to allow testing correct block // reassembly. func makeChain(n int, seed byte, parent *types.Block, empty bool) ([]*types.Block, []types.Receipts) { - blocks, receipts := core.GenerateChain(params.TestChainConfig, parent, ethash.NewFaker(), testdb, n, func(i int, block *core.BlockGen) { + blocks, receipts := core.GenerateChain(params.TestChainConfig, parent, ethash.NewFaker(), testDB, n, func(i int, block *core.BlockGen) { block.SetCoinbase(common.Address{seed}) // Add one tx to every secondblock if !empty && i%2 == 0 { @@ -69,10 +63,10 @@ var emptyChain *chainData func init() { // Create a chain of blocks to import targetBlocks := 128 - blocks, _ := makeChain(targetBlocks, 0, genesis, false) + blocks, _ := makeChain(targetBlocks, 0, testGenesis, false) chain = &chainData{blocks, 0} - blocks, _ = makeChain(targetBlocks, 0, genesis, true) + blocks, _ = makeChain(targetBlocks, 0, testGenesis, true) emptyChain = &chainData{blocks, 0} } @@ -150,7 +144,7 @@ func TestBasics(t *testing.T) { // The second peer should hit throttling if !throttle { - t.Fatalf("should not throttle") + t.Fatalf("should throttle") } // And not get any fetches at all, since it was throttled to begin with if fetchReq != nil { @@ -179,7 +173,6 @@ func TestBasics(t *testing.T) { if got, exp := fetchReq.Headers[0].Number.Uint64(), uint64(1); got != exp { t.Fatalf("expected header %d, got %d", exp, got) } - } if exp, got := q.blockTaskQueue.Size(), numOfBlocks-10; exp != got { t.Errorf("expected block task queue to be %d, got %d", exp, got) @@ -227,7 +220,6 @@ func TestEmptyBlocks(t *testing.T) { if fetchReq != nil { t.Fatal("there should be no body fetch tasks remaining") } - } if q.blockTaskQueue.Size() != numOfBlocks-10 { t.Errorf("expected block task queue to be %d, got %d", numOfBlocks-10, q.blockTaskQueue.Size()) @@ -241,7 +233,7 @@ func TestEmptyBlocks(t *testing.T) { // there should be nothing to fetch, blocks are empty if fetchReq != nil { - t.Fatal("there should be no body fetch tasks remaining") + t.Fatal("there should be no receipt fetch tasks remaining") } } if q.blockTaskQueue.Size() != numOfBlocks-10 { @@ -261,14 +253,13 @@ func TestEmptyBlocks(t *testing.T) { // some more advanced scenarios func XTestDelivery(t *testing.T) { // the outside network, holding blocks - blo, rec := makeChain(128, 0, genesis, false) + blo, rec := makeChain(128, 0, testGenesis, false) world := newNetwork() world.receipts = rec world.chain = blo world.progress(10) if false { log.Root().SetHandler(log.StdoutHandler) - } q := newQueue(10, 10) var wg sync.WaitGroup @@ -299,7 +290,6 @@ func XTestDelivery(t *testing.T) { fmt.Printf("got %d results, %d tot\n", len(res), tot) // Now we can forget about these world.forget(res[len(res)-1].Header.Number.Uint64()) - } }() wg.Add(1) @@ -362,7 +352,6 @@ func XTestDelivery(t *testing.T) { } for i := 0; i < 50; i++ { time.Sleep(2990 * time.Millisecond) - } }() wg.Add(1) @@ -413,10 +402,8 @@ func (n *network) forget(blocknum uint64) { n.chain = n.chain[index:] n.receipts = n.receipts[index:] n.offset = int(blocknum) - } func (n *network) progress(numBlocks int) { - n.lock.Lock() defer n.lock.Unlock() //fmt.Printf("progressing...\n") @@ -424,7 +411,6 @@ func (n *network) progress(numBlocks int) { n.chain = append(n.chain, newBlocks...) n.receipts = append(n.receipts, newR...) n.cond.Broadcast() - } func (n *network) headers(from int) []*types.Header { diff --git a/les/downloader/statesync.go b/les/downloader/statesync.go index 2b3278822996b..22f952155f119 100644 --- a/les/downloader/statesync.go +++ b/les/downloader/statesync.go @@ -34,7 +34,7 @@ import ( // a single data retrieval network packet. type stateReq struct { nItems uint16 // Number of items requested for download (max is 384, so uint16 is sufficient) - trieTasks map[common.Hash]*trieTask // Trie node download tasks to track previous attempts + trieTasks map[string]*trieTask // Trie node download tasks to track previous attempts codeTasks map[common.Hash]*codeTask // Byte code download tasks to track previous attempts timeout time.Duration // Maximum round trip time for this to complete timer *time.Timer // Timer to fire when the RTT timeout expires @@ -263,8 +263,8 @@ type stateSync struct { sched *trie.Sync // State trie sync scheduler defining the tasks keccak crypto.KeccakState // Keccak256 hasher to verify deliveries with - trieTasks map[common.Hash]*trieTask // Set of trie node tasks currently queued for retrieval - codeTasks map[common.Hash]*codeTask // Set of byte code tasks currently queued for retrieval + trieTasks map[string]*trieTask // Set of trie node tasks currently queued for retrieval, indexed by path + codeTasks map[common.Hash]*codeTask // Set of byte code tasks currently queued for retrieval, indexed by hash numUncommitted int bytesUncommitted int @@ -281,6 +281,7 @@ type stateSync struct { // trieTask represents a single trie node download task, containing a set of // peers already attempted retrieval from to detect stalled syncs and abort. type trieTask struct { + hash common.Hash path [][]byte attempts map[string]struct{} } @@ -299,7 +300,7 @@ func newStateSync(d *Downloader, root common.Hash) *stateSync { root: root, sched: state.NewStateSync(root, d.stateDB, nil), keccak: sha3.NewLegacyKeccak256().(crypto.KeccakState), - trieTasks: make(map[common.Hash]*trieTask), + trieTasks: make(map[string]*trieTask), codeTasks: make(map[common.Hash]*codeTask), deliver: make(chan *stateReq), cancel: make(chan struct{}), @@ -455,10 +456,11 @@ func (s *stateSync) assignTasks() { func (s *stateSync) fillTasks(n int, req *stateReq) (nodes []common.Hash, paths []trie.SyncPath, codes []common.Hash) { // Refill available tasks from the scheduler. if fill := n - (len(s.trieTasks) + len(s.codeTasks)); fill > 0 { - nodes, paths, codes := s.sched.Missing(fill) - for i, hash := range nodes { - s.trieTasks[hash] = &trieTask{ - path: paths[i], + paths, hashes, codes := s.sched.Missing(fill) + for i, path := range paths { + s.trieTasks[path] = &trieTask{ + hash: hashes[i], + path: trie.NewSyncPath([]byte(path)), attempts: make(map[string]struct{}), } } @@ -474,7 +476,7 @@ func (s *stateSync) fillTasks(n int, req *stateReq) (nodes []common.Hash, paths paths = make([]trie.SyncPath, 0, n) codes = make([]common.Hash, 0, n) - req.trieTasks = make(map[common.Hash]*trieTask, n) + req.trieTasks = make(map[string]*trieTask, n) req.codeTasks = make(map[common.Hash]*codeTask, n) for hash, t := range s.codeTasks { @@ -492,7 +494,7 @@ func (s *stateSync) fillTasks(n int, req *stateReq) (nodes []common.Hash, paths req.codeTasks[hash] = t delete(s.codeTasks, hash) } - for hash, t := range s.trieTasks { + for path, t := range s.trieTasks { // Stop when we've gathered enough requests if len(nodes)+len(codes) == n { break @@ -504,11 +506,11 @@ func (s *stateSync) fillTasks(n int, req *stateReq) (nodes []common.Hash, paths // Assign the request to this peer t.attempts[req.peer.id] = struct{}{} - nodes = append(nodes, hash) + nodes = append(nodes, t.hash) paths = append(paths, t.path) - req.trieTasks[hash] = t - delete(s.trieTasks, hash) + req.trieTasks[path] = t + delete(s.trieTasks, path) } req.nItems = uint16(len(nodes) + len(codes)) return nodes, paths, codes @@ -530,7 +532,7 @@ func (s *stateSync) process(req *stateReq) (int, error) { // Iterate over all the delivered data and inject one-by-one into the trie for _, blob := range req.response { - hash, err := s.processNodeData(blob) + hash, err := s.processNodeData(req.trieTasks, req.codeTasks, blob) switch err { case nil: s.numUncommitted++ @@ -543,13 +545,10 @@ func (s *stateSync) process(req *stateReq) (int, error) { default: return successful, fmt.Errorf("invalid state node %s: %v", hash.TerminalString(), err) } - // Delete from both queues (one delivery is enough for the syncer) - delete(req.trieTasks, hash) - delete(req.codeTasks, hash) } // Put unfulfilled tasks back into the retry queue npeers := s.d.peers.Len() - for hash, task := range req.trieTasks { + for path, task := range req.trieTasks { // If the node did deliver something, missing items may be due to a protocol // limit or a previous timeout + delayed delivery. Both cases should permit // the node to retry the missing items (to avoid single-peer stalls). @@ -559,10 +558,10 @@ func (s *stateSync) process(req *stateReq) (int, error) { // If we've requested the node too many times already, it may be a malicious // sync where nobody has the right data. Abort. if len(task.attempts) >= npeers { - return successful, fmt.Errorf("trie node %s failed with all peers (%d tries, %d peers)", hash.TerminalString(), len(task.attempts), npeers) + return successful, fmt.Errorf("trie node %s failed with all peers (%d tries, %d peers)", task.hash.TerminalString(), len(task.attempts), npeers) } // Missing item, place into the retry queue. - s.trieTasks[hash] = task + s.trieTasks[path] = task } for hash, task := range req.codeTasks { // If the node did deliver something, missing items may be due to a protocol @@ -585,13 +584,35 @@ func (s *stateSync) process(req *stateReq) (int, error) { // processNodeData tries to inject a trie node data blob delivered from a remote // peer into the state trie, returning whether anything useful was written or any // error occurred. -func (s *stateSync) processNodeData(blob []byte) (common.Hash, error) { - res := trie.SyncResult{Data: blob} +// +// If multiple requests correspond to the same hash, this method will inject the +// blob as a result for the first one only, leaving the remaining duplicates to +// be fetched again. +func (s *stateSync) processNodeData(nodeTasks map[string]*trieTask, codeTasks map[common.Hash]*codeTask, blob []byte) (common.Hash, error) { + var hash common.Hash s.keccak.Reset() s.keccak.Write(blob) - s.keccak.Read(res.Hash[:]) - err := s.sched.Process(res) - return res.Hash, err + s.keccak.Read(hash[:]) + + if _, present := codeTasks[hash]; present { + err := s.sched.ProcessCode(trie.CodeSyncResult{ + Hash: hash, + Data: blob, + }) + delete(codeTasks, hash) + return hash, err + } + for path, task := range nodeTasks { + if task.hash == hash { + err := s.sched.ProcessNode(trie.NodeSyncResult{ + Path: path, + Data: blob, + }) + delete(nodeTasks, path) + return hash, err + } + } + return common.Hash{}, trie.ErrNotRequested } // updateStats bumps the various state sync progress counters and displays a log @@ -608,7 +629,7 @@ func (s *stateSync) updateStats(written, duplicate, unexpected int, duration tim if written > 0 || duplicate > 0 || unexpected > 0 { log.Info("Imported new state entries", "count", written, "elapsed", common.PrettyDuration(duration), "processed", s.d.syncStatsState.processed, "pending", s.d.syncStatsState.pending, "trieretry", len(s.trieTasks), "coderetry", len(s.codeTasks), "duplicate", s.d.syncStatsState.duplicate, "unexpected", s.d.syncStatsState.unexpected) } - if written > 0 { - //rawdb.WriteFastTrieProgress(s.d.stateDB, s.d.syncStatsState.processed) - } + //if written > 0 { + //rawdb.WriteFastTrieProgress(s.d.stateDB, s.d.syncStatsState.processed) + //} } diff --git a/les/downloader/testchain_test.go b/les/downloader/testchain_test.go index b9865f7e032b3..400eec94e7c49 100644 --- a/les/downloader/testchain_test.go +++ b/les/downloader/testchain_test.go @@ -35,7 +35,12 @@ var ( testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") testAddress = crypto.PubkeyToAddress(testKey.PublicKey) testDB = rawdb.NewMemoryDatabase() - testGenesis = core.GenesisBlockForTesting(testDB, testAddress, big.NewInt(1000000000000000)) + + gspec = core.Genesis{ + Alloc: core.GenesisAlloc{testAddress: {Balance: big.NewInt(1000000000000000)}}, + BaseFee: big.NewInt(params.InitialBaseFee), + } + testGenesis = gspec.MustCommit(testDB) ) // The common prefix of all test chains: diff --git a/les/fetcher.go b/les/fetcher.go index cf62c8f707765..6861eebcf5e92 100644 --- a/les/fetcher.go +++ b/les/fetcher.go @@ -252,7 +252,7 @@ func (f *lightFetcher) forEachPeer(check func(id enode.ID, p *fetcherPeer) bool) // request will be made for header retrieval. // // - re-sync trigger -// If the local chain lags too much, then the fetcher will enter "synnchronise" +// If the local chain lags too much, then the fetcher will enter "synchronise" // mode to retrieve missing headers in batch. func (f *lightFetcher) mainloop() { defer f.wg.Done() diff --git a/les/fetcher/block_fetcher.go b/les/fetcher/block_fetcher.go index 283008db0f1e5..86b3c552ce27d 100644 --- a/les/fetcher/block_fetcher.go +++ b/les/fetcher/block_fetcher.go @@ -641,7 +641,6 @@ func (f *BlockFetcher) loop() { } else { f.forgetHash(hash) } - } if matched { task.transactions = append(task.transactions[:i], task.transactions[i+1:]...) diff --git a/les/fetcher/block_fetcher_test.go b/les/fetcher/block_fetcher_test.go index de066ac26b5ff..caff7a3b3559a 100644 --- a/les/fetcher/block_fetcher_test.go +++ b/les/fetcher/block_fetcher_test.go @@ -35,10 +35,15 @@ import ( ) var ( - testdb = rawdb.NewMemoryDatabase() - testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - testAddress = crypto.PubkeyToAddress(testKey.PublicKey) - genesis = core.GenesisBlockForTesting(testdb, testAddress, big.NewInt(1000000000000000)) + testdb = rawdb.NewMemoryDatabase() + testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + testAddress = crypto.PubkeyToAddress(testKey.PublicKey) + + gspec = core.Genesis{ + Alloc: core.GenesisAlloc{testAddress: {Balance: big.NewInt(1000000000000000)}}, + BaseFee: big.NewInt(params.InitialBaseFee), + } + genesis = gspec.MustCommit(testdb) unknownBlock = types.NewBlock(&types.Header{GasLimit: params.GenesisGasLimit, BaseFee: big.NewInt(params.InitialBaseFee)}, nil, nil, nil, trie.NewStackTrie(nil)) ) diff --git a/les/fetcher_test.go b/les/fetcher_test.go index 28db3b8913aca..6a17e73757a54 100644 --- a/les/fetcher_test.go +++ b/les/fetcher_test.go @@ -160,7 +160,6 @@ func testTrustedAnnouncement(t *testing.T, protocol int) { nodes []*enode.Node ids []string cpeers []*clientPeer - speers []*serverPeer config = light.TestServerIndexerConfig waitIndexers = func(cIndexer, bIndexer, btIndexer *core.ChainIndexer) { @@ -213,12 +212,11 @@ func testTrustedAnnouncement(t *testing.T, protocol int) { // Connect all server instances. for i := 0; i < len(servers); i++ { - sp, cp, err := connect(servers[i].handler, nodes[i].ID(), c.handler, protocol, true) + _, cp, err := connect(servers[i].handler, nodes[i].ID(), c.handler, protocol, true) if err != nil { t.Fatalf("connect server and client failed, err %s", err) } cpeers = append(cpeers, cp) - speers = append(speers, sp) } newHead := make(chan *types.Header, 1) c.handler.fetcher.newHeadHook = func(header *types.Header) { newHead <- header } diff --git a/les/flowcontrol/control.go b/les/flowcontrol/control.go index 4f0de82318357..76a241fa5a7f0 100644 --- a/les/flowcontrol/control.go +++ b/les/flowcontrol/control.go @@ -182,7 +182,7 @@ func (node *ClientNode) UpdateParams(params ServerParams) { return } } - node.updateSchedule = append(node.updateSchedule, scheduledUpdate{time: now + mclock.AbsTime(DecParamDelay), params: params}) + node.updateSchedule = append(node.updateSchedule, scheduledUpdate{time: now.Add(DecParamDelay), params: params}) } } diff --git a/les/flowcontrol/manager.go b/les/flowcontrol/manager.go index c9e681c1440a8..4367974d632e2 100644 --- a/les/flowcontrol/manager.go +++ b/les/flowcontrol/manager.go @@ -55,13 +55,12 @@ var ( // ClientManager controls the capacity assigned to the clients of a server. // Since ServerParams guarantee a safe lower estimate for processable requests // even in case of all clients being active, ClientManager calculates a -// corrigated buffer value and usually allows a higher remaining buffer value +// corrugated buffer value and usually allows a higher remaining buffer value // to be returned with each reply. type ClientManager struct { - clock mclock.Clock - lock sync.Mutex - enabledCh chan struct{} - stop chan chan struct{} + clock mclock.Clock + lock sync.Mutex + stop chan chan struct{} curve PieceWiseLinear sumRecharge, totalRecharge, totalConnected uint64 diff --git a/les/flowcontrol/manager_test.go b/les/flowcontrol/manager_test.go index 9d2f88763614a..564d813f15a38 100644 --- a/les/flowcontrol/manager_test.go +++ b/les/flowcontrol/manager_test.go @@ -104,7 +104,6 @@ func testConstantTotalCapacity(t *testing.T, nodeCount, maxCapacityNodes, random if ratio < 0.98 || ratio > 1.02 { t.Errorf("totalCost/totalCapacity/testLength ratio incorrect (expected: 1, got: %f)", ratio) } - } func (n *testNode) send(t *testing.T, now mclock.AbsTime) bool { diff --git a/les/handler_test.go b/les/handler_test.go index aba45764b3068..56d7d55b5a5f7 100644 --- a/les/handler_test.go +++ b/les/handler_test.go @@ -405,7 +405,7 @@ func testGetProofs(t *testing.T, protocol int) { accounts := []common.Address{bankAddr, userAddr1, userAddr2, signerAddr, {}} for i := uint64(0); i <= bc.CurrentBlock().NumberU64(); i++ { header := bc.GetHeaderByNumber(i) - trie, _ := trie.New(header.Root, trie.NewDatabase(server.db)) + trie, _ := trie.New(common.Hash{}, header.Root, trie.NewDatabase(server.db)) for _, acc := range accounts { req := ProofReq{ @@ -456,7 +456,7 @@ func testGetStaleProof(t *testing.T, protocol int) { var expected []rlp.RawValue if wantOK { proofsV2 := light.NewNodeSet() - t, _ := trie.New(header.Root, trie.NewDatabase(server.db)) + t, _ := trie.New(common.Hash{}, header.Root, trie.NewDatabase(server.db)) t.Prove(account, 0, proofsV2) expected = proofsV2.NodeList() } @@ -512,7 +512,7 @@ func testGetCHTProofs(t *testing.T, protocol int) { AuxData: [][]byte{rlp}, } root := light.GetChtRoot(server.db, 0, bc.GetHeaderByNumber(config.ChtSize-1).Hash()) - trie, _ := trie.New(root, trie.NewDatabase(rawdb.NewTable(server.db, light.ChtTablePrefix))) + trie, _ := trie.New(common.Hash{}, root, trie.NewDatabase(rawdb.NewTable(server.db, light.ChtTablePrefix))) trie.Prove(key, 0, &proofsV2.Proofs) // Assemble the requests for the different protocols requestsV2 := []HelperTrieReq{{ @@ -577,7 +577,7 @@ func testGetBloombitsProofs(t *testing.T, protocol int) { var proofs HelperTrieResps root := light.GetBloomTrieRoot(server.db, 0, bc.GetHeaderByNumber(config.BloomTrieSize-1).Hash()) - trie, _ := trie.New(root, trie.NewDatabase(rawdb.NewTable(server.db, light.BloomTrieTablePrefix))) + trie, _ := trie.New(common.Hash{}, root, trie.NewDatabase(rawdb.NewTable(server.db, light.BloomTrieTablePrefix))) trie.Prove(key, 0, &proofs.Proofs) // Send the proof request and verify the response diff --git a/les/odr.go b/les/odr.go index 10ff0854d3850..2643a534787f5 100644 --- a/les/odr.go +++ b/les/odr.go @@ -126,7 +126,7 @@ const ( // RetrieveTxStatus retrieves the transaction status from the LES network. // There is no guarantee in the LES protocol that the mined transaction will // be retrieved back for sure because of different reasons(the transaction -// is unindexed, the malicous server doesn't reply it deliberately, etc). +// is unindexed, the malicious server doesn't reply it deliberately, etc). // Therefore, unretrieved transactions(UNKNOWN) will receive a certain number // of retries, thus giving a weak guarantee. func (odr *LesOdr) RetrieveTxStatus(ctx context.Context, req *light.TxStatusRequest) error { diff --git a/les/odr_test.go b/les/odr_test.go index ad77abf5b9b28..24b8e2ae31111 100644 --- a/les/odr_test.go +++ b/les/odr_test.go @@ -392,12 +392,10 @@ func testGetTxStatusFromUnindexedPeers(t *testing.T, protocol int) { for _, testspec := range testspecs { // Create a bunch of server peers with different tx history var ( - serverPeers []*testPeer - closeFns []func() + closeFns []func() ) for i := 0; i < testspec.peers; i++ { peer, closePeer, _ := client.newRawPeer(t, fmt.Sprintf("server-%d", i), protocol, testspec.txLookups[i]) - serverPeers = append(serverPeers, peer) closeFns = append(closeFns, closePeer) // Create a one-time routine for serving message diff --git a/les/peer.go b/les/peer.go index 499429739d23d..deda052a3b14b 100644 --- a/les/peer.go +++ b/les/peer.go @@ -995,40 +995,6 @@ func (p *clientPeer) sendLastAnnounce() { } } -// freezeClient temporarily puts the client in a frozen state which means all -// unprocessed and subsequent requests are dropped. Unfreezing happens automatically -// after a short time if the client's buffer value is at least in the slightly positive -// region. The client is also notified about being frozen/unfrozen with a Stop/Resume -// message. -func (p *clientPeer) freezeClient() { - if p.version < lpv3 { - // if Stop/Resume is not supported then just drop the peer after setting - // its frozen status permanently - atomic.StoreUint32(&p.frozen, 1) - p.Peer.Disconnect(p2p.DiscUselessPeer) - return - } - if atomic.SwapUint32(&p.frozen, 1) == 0 { - go func() { - p.sendStop() - time.Sleep(freezeTimeBase + time.Duration(rand.Int63n(int64(freezeTimeRandom)))) - for { - bufValue, bufLimit := p.fcClient.BufferStatus() - if bufLimit == 0 { - return - } - if bufValue <= bufLimit/8 { - time.Sleep(freezeCheckPeriod) - } else { - atomic.StoreUint32(&p.frozen, 0) - p.sendResume(bufValue) - break - } - } - }() - } -} - // Handshake executes the les protocol handshake, negotiating version number, // network IDs, difficulties, head and genesis blocks. func (p *clientPeer) Handshake(td *big.Int, head common.Hash, headNum uint64, genesis common.Hash, forkID forkid.ID, forkFilter forkid.Filter, server *LesServer) error { @@ -1157,19 +1123,6 @@ func (ps *serverPeerSet) subscribe(sub serverPeerSubscriber) { } } -// unSubscribe removes the specified service from the subscriber pool. -func (ps *serverPeerSet) unSubscribe(sub serverPeerSubscriber) { - ps.lock.Lock() - defer ps.lock.Unlock() - - for i, s := range ps.subscribers { - if s == sub { - ps.subscribers = append(ps.subscribers[:i], ps.subscribers[i+1:]...) - return - } - } -} - // register adds a new server peer into the set, or returns an error if the // peer is already known. func (ps *serverPeerSet) register(peer *serverPeer) error { @@ -1236,25 +1189,6 @@ func (ps *serverPeerSet) len() int { return len(ps.peers) } -// bestPeer retrieves the known peer with the currently highest total difficulty. -// If the peerset is "client peer set", then nothing meaningful will return. The -// reason is client peer never send back their latest status to server. -func (ps *serverPeerSet) bestPeer() *serverPeer { - ps.lock.RLock() - defer ps.lock.RUnlock() - - var ( - bestPeer *serverPeer - bestTd *big.Int - ) - for _, p := range ps.peers { - if td := p.Td(); bestTd == nil || td.Cmp(bestTd) > 0 { - bestPeer, bestTd = p, td - } - } - return bestPeer -} - // allServerPeers returns all server peers in a list. func (ps *serverPeerSet) allPeers() []*serverPeer { ps.lock.RLock() @@ -1348,14 +1282,6 @@ func (ps *clientPeerSet) peer(id enode.ID) *clientPeer { return ps.peers[id] } -// len returns if the current number of peers in the set. -func (ps *clientPeerSet) len() int { - ps.lock.RLock() - defer ps.lock.RUnlock() - - return len(ps.peers) -} - // setSignerKey sets the signer key for signed announcements. Should be called before // starting the protocol handler. func (ps *clientPeerSet) setSignerKey(privateKey *ecdsa.PrivateKey) { diff --git a/les/peer_test.go b/les/peer_test.go index d6551ce6b6393..b8a1482a040a9 100644 --- a/les/peer_test.go +++ b/les/peer_test.go @@ -28,7 +28,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/forkid" - "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" @@ -100,7 +99,7 @@ type fakeChain struct{} func (f *fakeChain) Config() *params.ChainConfig { return params.MainnetChainConfig } func (f *fakeChain) Genesis() *types.Block { - return core.DefaultGenesisBlock().ToBlock(rawdb.NewMemoryDatabase()) + return core.DefaultGenesisBlock().ToBlock() } func (f *fakeChain) CurrentHeader() *types.Header { return &types.Header{Number: big.NewInt(10000000)} } diff --git a/les/server.go b/les/server.go index c135e65f2dc4b..df453b4819a2e 100644 --- a/les/server.go +++ b/les/server.go @@ -159,21 +159,15 @@ func (s *LesServer) APIs() []rpc.API { return []rpc.API{ { Namespace: "les", - Version: "1.0", - Service: NewPrivateLightAPI(&s.lesCommons), - Public: false, + Service: NewLightAPI(&s.lesCommons), }, { Namespace: "les", - Version: "1.0", - Service: NewPrivateLightServerAPI(s), - Public: false, + Service: NewLightServerAPI(s), }, { Namespace: "debug", - Version: "1.0", - Service: NewPrivateDebugAPI(s), - Public: false, + Service: NewDebugAPI(s), }, } } diff --git a/les/server_handler.go b/les/server_handler.go index ef1af844c26b1..a199a34a72545 100644 --- a/les/server_handler.go +++ b/les/server_handler.go @@ -359,7 +359,7 @@ func (h *serverHandler) AddTxsSync() bool { // getAccount retrieves an account from the state based on root. func getAccount(triedb *trie.Database, root, hash common.Hash) (types.StateAccount, error) { - trie, err := trie.New(root, triedb) + trie, err := trie.New(common.Hash{}, root, triedb) if err != nil { return types.StateAccount{}, err } @@ -391,7 +391,7 @@ func (h *serverHandler) GetHelperTrie(typ uint, index uint64) *trie.Trie { if root == (common.Hash{}) { return nil } - trie, _ := trie.New(root, trie.NewDatabase(rawdb.NewTable(h.chainDb, prefix))) + trie, _ := trie.New(common.Hash{}, root, trie.NewDatabase(rawdb.NewTable(h.chainDb, prefix))) return trie } diff --git a/les/ulc_test.go b/les/ulc_test.go index a4df0795b46d4..9a29a24cee55b 100644 --- a/les/ulc_test.go +++ b/les/ulc_test.go @@ -35,6 +35,19 @@ func TestULCAnnounceThresholdLes3(t *testing.T) { testULCAnnounceThreshold(t, 3) func testULCAnnounceThreshold(t *testing.T, protocol int) { // todo figure out why it takes fetcher so longer to fetcher the announced header. t.Skip("Sometimes it can failed") + + // newTestLightPeer creates node with light sync mode + newTestLightPeer := func(t *testing.T, protocol int, ulcServers []string, ulcFraction int) (*testClient, func()) { + netconfig := testnetConfig{ + protocol: protocol, + ulcServers: ulcServers, + ulcFraction: ulcFraction, + nopruning: true, + } + _, c, teardown := newClientServerEnv(t, netconfig) + return c, teardown + } + var cases = []struct { height []int threshold int @@ -148,15 +161,3 @@ func newTestServerPeer(t *testing.T, blocks int, protocol int, indexFn indexerCa n := enode.NewV4(&key.PublicKey, net.ParseIP("127.0.0.1"), 35000, 35000) return s, n, teardown } - -// newTestLightPeer creates node with light sync mode -func newTestLightPeer(t *testing.T, protocol int, ulcServers []string, ulcFraction int) (*testClient, func()) { - netconfig := testnetConfig{ - protocol: protocol, - ulcServers: ulcServers, - ulcFraction: ulcFraction, - nopruning: true, - } - _, c, teardown := newClientServerEnv(t, netconfig) - return c, teardown -} diff --git a/les/utils/timeutils_test.go b/les/utils/timeutils_test.go index 9f9e1c2dc9389..b219d0439dcb2 100644 --- a/les/utils/timeutils_test.go +++ b/les/utils/timeutils_test.go @@ -37,7 +37,7 @@ func TestUpdateTimer(t *testing.T) { if updated := timer.Update(func(diff time.Duration) bool { return true }); !updated { t.Fatalf("Doesn't update the clock when reaching the threshold") } - if updated := timer.UpdateAt(sim.Now()+mclock.AbsTime(time.Second), func(diff time.Duration) bool { return true }); !updated { + if updated := timer.UpdateAt(sim.Now().Add(time.Second), func(diff time.Duration) bool { return true }); !updated { t.Fatalf("Doesn't update the clock when reaching the threshold") } timer = NewUpdateTimer(sim, 0) diff --git a/les/vflux/client/fillset_test.go b/les/vflux/client/fillset_test.go index ca5af8f07ecc9..ddb12a82f9b3a 100644 --- a/les/vflux/client/fillset_test.go +++ b/les/vflux/client/fillset_test.go @@ -104,7 +104,7 @@ func TestFillSet(t *testing.T) { fs.SetTarget(10) expWaiting(4, true) expNotWaiting() - // remove all previosly set flags + // remove all previously set flags ns.ForEach(sfTest1, nodestate.Flags{}, func(node *enode.Node, state nodestate.Flags) { ns.SetState(node, nodestate.Flags{}, sfTest1, 0) }) diff --git a/les/vflux/client/serverpool.go b/les/vflux/client/serverpool.go index e481075f70bd8..cf96f0ee3a23b 100644 --- a/les/vflux/client/serverpool.go +++ b/les/vflux/client/serverpool.go @@ -222,7 +222,6 @@ func (s *serverPoolIterator) Close() { func (s *ServerPool) AddMetrics( suggestedTimeoutGauge, totalValueGauge, serverSelectableGauge, serverConnectedGauge metrics.Gauge, sessionValueMeter, serverDialedMeter metrics.Meter) { - s.suggestedTimeoutGauge = suggestedTimeoutGauge s.totalValueGauge = totalValueGauge s.sessionValueMeter = sessionValueMeter diff --git a/les/vflux/client/serverpool_test.go b/les/vflux/client/serverpool_test.go index c7d0245ef21d9..f1fd987d7edb1 100644 --- a/les/vflux/client/serverpool_test.go +++ b/les/vflux/client/serverpool_test.go @@ -55,7 +55,6 @@ type ServerPoolTest struct { clock *mclock.Simulated quit chan chan struct{} preNeg, preNegFail bool - vt *ValueTracker sp *ServerPool spi enode.Iterator input enode.Iterator @@ -67,7 +66,7 @@ type ServerPoolTest struct { // (accessed from both the main thread and the preNeg callback) preNegLock sync.Mutex queryWg *sync.WaitGroup // a new wait group is created each time the simulation is started - stopping bool // stopping avoid callind queryWg.Add after queryWg.Wait + stopping bool // stopping avoid calling queryWg.Add after queryWg.Wait cycle, conn, servedConn int serviceCycles, dialCount int diff --git a/les/vflux/client/wrsiterator.go b/les/vflux/client/wrsiterator.go index 8a2e39ad44228..1b37cba6e5ded 100644 --- a/les/vflux/client/wrsiterator.go +++ b/les/vflux/client/wrsiterator.go @@ -109,7 +109,6 @@ func (w *WrsIterator) chooseNode() *enode.Node { return w.ns.GetNode(id) } } - } // Close ends the iterator. diff --git a/les/vflux/server/balance.go b/les/vflux/server/balance.go index 727ce09a432fc..b09f7bb5012b0 100644 --- a/les/vflux/server/balance.go +++ b/les/vflux/server/balance.go @@ -356,7 +356,7 @@ func (n *nodeBalance) estimatePriority(capacity uint64, addBalance int64, future b = n.reducedBalance(b, now, future, capacity, avgReqCost) } if bias > 0 { - b = n.reducedBalance(b, now+mclock.AbsTime(future), bias, capacity, 0) + b = n.reducedBalance(b, now.Add(future), bias, capacity, 0) } pri := n.balanceToPriority(now, b, capacity) // Ensure that biased estimates are always lower than actual priorities, even if @@ -512,7 +512,7 @@ func (n *nodeBalance) scheduleCheck(now mclock.AbsTime) { n.updateAfter(0) return } - if n.nextUpdate == 0 || n.nextUpdate > now+mclock.AbsTime(d) { + if n.nextUpdate == 0 || n.nextUpdate > now.Add(d) { if d > time.Second { // Note: if the scheduled update is not in the very near future then we // schedule the update a bit earlier. This way we do need to update a few @@ -520,7 +520,7 @@ func (n *nodeBalance) scheduleCheck(now mclock.AbsTime) { // brings the expected firing time a little bit closer. d = ((d - time.Second) * 7 / 8) + time.Second } - n.nextUpdate = now + mclock.AbsTime(d) + n.nextUpdate = now.Add(d) n.updateAfter(d) } } else { @@ -623,13 +623,13 @@ func (n *nodeBalance) priorityToBalance(priority int64, capacity uint64) (uint64 return 0, uint64(-priority) } -// reducedBalance estimates the reduced balance at a given time in the fututre based +// reducedBalance estimates the reduced balance at a given time in the future based // on the given balance, the time factor and an estimated average request cost per time ratio func (n *nodeBalance) reducedBalance(b balance, start mclock.AbsTime, dt time.Duration, capacity uint64, avgReqCost float64) balance { // since the costs are applied continuously during the dt time period we calculate // the expiration offset at the middle of the period var ( - at = start + mclock.AbsTime(dt/2) + at = start.Add(dt / 2) dtf = float64(dt) ) if !b.pos.IsZero() { diff --git a/les/vflux/server/balance_test.go b/les/vflux/server/balance_test.go index 9f253cabf48db..7c100aab509fa 100644 --- a/les/vflux/server/balance_test.go +++ b/les/vflux/server/balance_test.go @@ -54,7 +54,7 @@ func newBalanceTestSetup(db ethdb.KeyValueStore, posExp, negExp utils.ValueExpir // Initialize and customize the setup for the balance testing clock := &mclock.Simulated{} setup := newServerSetup() - setup.clientField = setup.setup.NewField("balancTestClient", reflect.TypeOf(balanceTestClient{})) + setup.clientField = setup.setup.NewField("balanceTestClient", reflect.TypeOf(balanceTestClient{})) ns := nodestate.NewNodeStateMachine(nil, nil, clock, setup.setup) if posExp == nil { @@ -298,7 +298,7 @@ func TestEstimatedPriority(t *testing.T) { } } -func TestPostiveBalanceCounting(t *testing.T) { +func TestPositiveBalanceCounting(t *testing.T) { b := newBalanceTestSetup(nil, nil, nil) defer b.stop() diff --git a/les/vflux/server/clientpool.go b/les/vflux/server/clientpool.go index e90469bb1c9a4..734d74f453c93 100644 --- a/les/vflux/server/clientpool.go +++ b/les/vflux/server/clientpool.go @@ -61,7 +61,6 @@ type ClientPool struct { setup *serverSetup clock mclock.Clock - closed bool ns *nodestate.NodeStateMachine synced func() bool diff --git a/les/vflux/server/clientpool_test.go b/les/vflux/server/clientpool_test.go index 49e66297a1b1a..790ec51360781 100644 --- a/les/vflux/server/clientpool_test.go +++ b/les/vflux/server/clientpool_test.go @@ -410,7 +410,6 @@ func TestFreeClientKickedOut(t *testing.T) { clock.Run(5 * time.Minute) for i := 0; i < 10; i++ { connect(pool, newPoolTestPeer(i+10, kicked)) - } clock.Run(0) diff --git a/les/vflux/server/status.go b/les/vflux/server/status.go index 469190777b252..2d7e25b684610 100644 --- a/les/vflux/server/status.go +++ b/les/vflux/server/status.go @@ -41,7 +41,7 @@ type serverSetup struct { activeFlag nodestate.Flags // Flag is set if the node is active inactiveFlag nodestate.Flags // Flag is set if the node is inactive capacityField nodestate.Field // Field contains the capacity of the node - queueField nodestate.Field // Field contains the infomration in the priority queue + queueField nodestate.Field // Field contains the information in the priority queue } // newServerSetup initializes the setup for state machine and returns the flags/fields group. diff --git a/light/lightchain.go b/light/lightchain.go index fa0dc71c95996..dca97ce45ce6a 100644 --- a/light/lightchain.go +++ b/light/lightchain.go @@ -355,22 +355,6 @@ func (lc *LightChain) Rollback(chain []common.Hash) { } } -// postChainEvents iterates over the events generated by a chain insertion and -// posts them into the event feed. -func (lc *LightChain) postChainEvents(events []interface{}) { - for _, event := range events { - switch ev := event.(type) { - case core.ChainEvent: - if lc.CurrentHeader().Hash() == ev.Hash { - lc.chainHeadFeed.Send(core.ChainHeadEvent{Block: ev.Block}) - } - lc.chainFeed.Send(ev) - case core.ChainSideEvent: - lc.chainSideFeed.Send(ev) - } - } -} - func (lc *LightChain) InsertHeader(header *types.Header) error { // Verify the header first before obtaining the lock headers := []*types.Header{header} @@ -413,7 +397,7 @@ func (lc *LightChain) SetCanonical(header *types.Header) error { // // The verify parameter can be used to fine tune whether nonce verification // should be done or not. The reason behind the optional check is because some -// of the header retrieval mechanisms already need to verfy nonces, as well as +// of the header retrieval mechanisms already need to verify nonces, as well as // because nonces can be verified sparsely, not needing to check each. // // In the case of a light chain, InsertHeaderChain also creates and posts light diff --git a/light/odr_test.go b/light/odr_test.go index fdf657a82ec5d..ec109f61f2bf5 100644 --- a/light/odr_test.go +++ b/light/odr_test.go @@ -82,7 +82,7 @@ func (odr *testOdr) Retrieve(ctx context.Context, req OdrRequest) error { req.Receipts = rawdb.ReadRawReceipts(odr.sdb, req.Hash, *number) } case *TrieRequest: - t, _ := trie.New(req.Id.Root, trie.NewDatabase(odr.sdb)) + t, _ := trie.New(common.BytesToHash(req.Id.AccKey), req.Id.Root, trie.NewDatabase(odr.sdb)) nodes := NewNodeSet() t.Prove(req.Key, 0, nodes) req.Proof = nodes diff --git a/light/odr_util.go b/light/odr_util.go index bbbcdbce21359..48631139b488a 100644 --- a/light/odr_util.go +++ b/light/odr_util.go @@ -272,9 +272,9 @@ func GetBloomBits(ctx context.Context, odr OdrBackend, bit uint, sections []uint // GetTransaction retrieves a canonical transaction by hash and also returns // its position in the chain. There is no guarantee in the LES protocol that // the mined transaction will be retrieved back for sure because of different -// reasons(the transaction is unindexed, the malicous server doesn't reply it +// reasons(the transaction is unindexed, the malicious server doesn't reply it // deliberately, etc). Therefore, unretrieved transactions will receive a certain -// number of retrys, thus giving a weak guarantee. +// number of retries, thus giving a weak guarantee. func GetTransaction(ctx context.Context, odr OdrBackend, txHash common.Hash) (*types.Transaction, common.Hash, uint64, uint64, error) { r := &TxStatusRequest{Hashes: []common.Hash{txHash}} if err := odr.RetrieveTxStatus(ctx, r); err != nil || r.Status[0].Status != core.TxStatusIncluded { diff --git a/light/postprocess.go b/light/postprocess.go index ce38d091e891a..3f9da659333ec 100644 --- a/light/postprocess.go +++ b/light/postprocess.go @@ -187,12 +187,12 @@ func (c *ChtIndexerBackend) Reset(ctx context.Context, section uint64, lastSecti root = GetChtRoot(c.diskdb, section-1, lastSectionHead) } var err error - c.trie, err = trie.New(root, c.triedb) + c.trie, err = trie.New(common.Hash{}, root, c.triedb) if err != nil && c.odr != nil { err = c.fetchMissingNodes(ctx, section, root) if err == nil { - c.trie, err = trie.New(root, c.triedb) + c.trie, err = trie.New(common.Hash{}, root, c.triedb) } } c.section = section @@ -217,7 +217,18 @@ func (c *ChtIndexerBackend) Process(ctx context.Context, header *types.Header) e // Commit implements core.ChainIndexerBackend func (c *ChtIndexerBackend) Commit() error { - root, _, err := c.trie.Commit(nil) + root, nodes, err := c.trie.Commit(false) + if err != nil { + return err + } + // Commit trie changes into trie database in case it's not nil. + if nodes != nil { + if err := c.triedb.Update(trie.NewWithNodeSet(nodes)); err != nil { + return err + } + } + // Re-create trie with newly generated root and updated database. + c.trie, err = trie.New(common.Hash{}, root, c.triedb) if err != nil { return err } @@ -253,9 +264,8 @@ func (c *ChtIndexerBackend) Commit() error { return nil } -// PruneSections implements core.ChainIndexerBackend which deletes all -// chain data(except hash<->number mappings) older than the specified -// threshold. +// Prune implements core.ChainIndexerBackend which deletes all chain data +// (except hash<->number mappings) older than the specified threshold. func (c *ChtIndexerBackend) Prune(threshold uint64) error { // Short circuit if the light pruning is disabled. if c.disablePruning { @@ -303,7 +313,7 @@ var ( BloomTrieTablePrefix = "blt-" ) -// GetBloomTrieRoot reads the BloomTrie root assoctiated to the given section from the database +// GetBloomTrieRoot reads the BloomTrie root associated to the given section from the database func GetBloomTrieRoot(db ethdb.Database, sectionIdx uint64, sectionHead common.Hash) common.Hash { var encNumber [8]byte binary.BigEndian.PutUint64(encNumber[:], sectionIdx) @@ -311,7 +321,7 @@ func GetBloomTrieRoot(db ethdb.Database, sectionIdx uint64, sectionHead common.H return common.BytesToHash(data) } -// StoreBloomTrieRoot writes the BloomTrie root assoctiated to the given section into the database +// StoreBloomTrieRoot writes the BloomTrie root associated to the given section into the database func StoreBloomTrieRoot(db ethdb.Database, sectionIdx uint64, sectionHead, root common.Hash) { var encNumber [8]byte binary.BigEndian.PutUint64(encNumber[:], sectionIdx) @@ -404,11 +414,11 @@ func (b *BloomTrieIndexerBackend) Reset(ctx context.Context, section uint64, las root = GetBloomTrieRoot(b.diskdb, section-1, lastSectionHead) } var err error - b.trie, err = trie.New(root, b.triedb) + b.trie, err = trie.New(common.Hash{}, root, b.triedb) if err != nil && b.odr != nil { err = b.fetchMissingNodes(ctx, section, root) if err == nil { - b.trie, err = trie.New(root, b.triedb) + b.trie, err = trie.New(common.Hash{}, root, b.triedb) } } b.section = section @@ -454,7 +464,18 @@ func (b *BloomTrieIndexerBackend) Commit() error { b.trie.Delete(encKey[:]) } } - root, _, err := b.trie.Commit(nil) + root, nodes, err := b.trie.Commit(false) + if err != nil { + return err + } + // Commit trie changes into trie database in case it's not nil. + if nodes != nil { + if err := b.triedb.Update(trie.NewWithNodeSet(nodes)); err != nil { + return err + } + } + // Re-create trie with newly generated root and updated database. + b.trie, err = trie.New(common.Hash{}, root, b.triedb) if err != nil { return err } diff --git a/light/trie.go b/light/trie.go index 4ab6f4ace0756..b88265e87d400 100644 --- a/light/trie.go +++ b/light/trie.go @@ -112,6 +112,22 @@ func (t *odrTrie) TryGet(key []byte) ([]byte, error) { return res, err } +func (t *odrTrie) TryGetAccount(key []byte) (*types.StateAccount, error) { + key = crypto.Keccak256(key) + var res types.StateAccount + err := t.do(key, func() (err error) { + value, err := t.trie.TryGet(key) + if err != nil { + return err + } + if value == nil { + return nil + } + return rlp.DecodeBytes(value, &res) + }) + return &res, err +} + func (t *odrTrie) TryUpdateAccount(key []byte, acc *types.StateAccount) error { key = crypto.Keccak256(key) value, err := rlp.EncodeToBytes(acc) @@ -137,11 +153,19 @@ func (t *odrTrie) TryDelete(key []byte) error { }) } -func (t *odrTrie) Commit(onleaf trie.LeafCallback) (common.Hash, int, error) { +// TryDeleteAccount abstracts an account deletion from the trie. +func (t *odrTrie) TryDeleteAccount(key []byte) error { + key = crypto.Keccak256(key) + return t.do(key, func() error { + return t.trie.TryDelete(key) + }) +} + +func (t *odrTrie) Commit(collectLeaf bool) (common.Hash, *trie.NodeSet, error) { if t.trie == nil { - return t.id.Root, 0, nil + return t.id.Root, nil, nil } - return t.trie.Commit(onleaf) + return t.trie.Commit(collectLeaf) } func (t *odrTrie) Hash() common.Hash { @@ -169,7 +193,11 @@ func (t *odrTrie) do(key []byte, fn func() error) error { for { var err error if t.trie == nil { - t.trie, err = trie.New(t.id.Root, trie.NewDatabase(t.db.backend.Database())) + var owner common.Hash + if len(t.id.AccKey) > 0 { + owner = common.BytesToHash(t.id.AccKey) + } + t.trie, err = trie.New(owner, t.id.Root, trie.NewDatabase(t.db.backend.Database())) } if err == nil { err = fn() @@ -195,7 +223,11 @@ func newNodeIterator(t *odrTrie, startkey []byte) trie.NodeIterator { // Open the actual non-ODR trie if that hasn't happened yet. if t.trie == nil { it.do(func() error { - t, err := trie.New(t.id.Root, trie.NewDatabase(t.db.backend.Database())) + var owner common.Hash + if len(t.id.AccKey) > 0 { + owner = common.BytesToHash(t.id.AccKey) + } + t, err := trie.New(owner, t.id.Root, trie.NewDatabase(t.db.backend.Database())) if err == nil { it.t.trie = t } diff --git a/light/trie_test.go b/light/trie_test.go index e8294cc2a2354..63dd9020f20c4 100644 --- a/light/trie_test.go +++ b/light/trie_test.go @@ -76,7 +76,7 @@ func diffTries(t1, t2 state.Trie) error { case i1.Err != nil: return fmt.Errorf("full trie iterator error: %v", i1.Err) case i2.Err != nil: - return fmt.Errorf("light trie iterator error: %v", i1.Err) + return fmt.Errorf("light trie iterator error: %v", i2.Err) case i1.Next(): return fmt.Errorf("full trie iterator has more k/v pairs") case i2.Next(): diff --git a/light/txpool.go b/light/txpool.go index a7df4aeec3884..b3e1a62e1893b 100644 --- a/light/txpool.go +++ b/light/txpool.go @@ -71,7 +71,7 @@ type TxPool struct { eip2718 bool // Fork indicator whether we are in the eip2718 stage. } -// TxRelayBackend provides an interface to the mechanism that forwards transacions +// TxRelayBackend provides an interface to the mechanism that forwards transactions // to the ETH network. The implementations of the functions should be non-blocking. // // Send instructs backend to forward new transactions @@ -398,7 +398,7 @@ func (pool *TxPool) add(ctx context.Context, tx *types.Transaction) error { hash := tx.Hash() if pool.pending[hash] != nil { - return fmt.Errorf("Known transaction (%x)", hash[:4]) + return fmt.Errorf("known transaction (%x)", hash[:4]) } err := pool.validateTx(ctx, tx) if err != nil { diff --git a/metrics/gauge_float64_test.go b/metrics/gauge_float64_test.go index 02b75580c4e5d..7b854d232ba8a 100644 --- a/metrics/gauge_float64_test.go +++ b/metrics/gauge_float64_test.go @@ -2,7 +2,7 @@ package metrics import "testing" -func BenchmarkGuageFloat64(b *testing.B) { +func BenchmarkGaugeFloat64(b *testing.B) { g := NewGaugeFloat64() b.ResetTimer() for i := 0; i < b.N; i++ { diff --git a/metrics/gauge_test.go b/metrics/gauge_test.go index 3aee143455c37..a98fe985d8c28 100644 --- a/metrics/gauge_test.go +++ b/metrics/gauge_test.go @@ -5,7 +5,7 @@ import ( "testing" ) -func BenchmarkGuage(b *testing.B) { +func BenchmarkGauge(b *testing.B) { g := NewGauge() b.ResetTimer() for i := 0; i < b.N; i++ { diff --git a/metrics/influxdb/influxdb.go b/metrics/influxdb/influxdb.go index dac9e824775a2..e99717aeebf9a 100644 --- a/metrics/influxdb/influxdb.go +++ b/metrics/influxdb/influxdb.go @@ -98,16 +98,16 @@ func (r *reporter) makeClient() (err error) { } func (r *reporter) run() { - intervalTicker := time.Tick(r.interval) - pingTicker := time.Tick(time.Second * 5) + intervalTicker := time.NewTicker(r.interval) + pingTicker := time.NewTicker(time.Second * 5) for { select { - case <-intervalTicker: + case <-intervalTicker.C: if err := r.send(); err != nil { log.Warn("Unable to send to InfluxDB", "err", err) } - case <-pingTicker: + case <-pingTicker.C: _, _, err := r.client.Ping() if err != nil { log.Warn("Got error while sending a ping to InfluxDB, trying to recreate client", "err", err) diff --git a/metrics/influxdb/influxdbv2.go b/metrics/influxdb/influxdbv2.go index 00901f52c9f4f..c8eca41616146 100644 --- a/metrics/influxdb/influxdbv2.go +++ b/metrics/influxdb/influxdbv2.go @@ -67,21 +67,20 @@ func InfluxDBV2WithTags(r metrics.Registry, d time.Duration, endpoint string, to } func (r *v2Reporter) run() { - intervalTicker := time.Tick(r.interval) - pingTicker := time.Tick(time.Second * 5) + intervalTicker := time.NewTicker(r.interval) + pingTicker := time.NewTicker(time.Second * 5) for { select { - case <-intervalTicker: + case <-intervalTicker.C: r.send() - case <-pingTicker: + case <-pingTicker.C: _, err := r.client.Health(context.Background()) if err != nil { log.Warn("Got error from influxdb client health check", "err", err.Error()) } } } - } func (r *v2Reporter) send() { @@ -90,7 +89,6 @@ func (r *v2Reporter) send() { namespace := r.namespace switch metric := i.(type) { - case metrics.Counter: v := metric.Count() l := r.cache[name] diff --git a/metrics/prometheus/prometheus.go b/metrics/prometheus/prometheus.go index 9ad5ec7e9929d..c8408d8cab851 100644 --- a/metrics/prometheus/prometheus.go +++ b/metrics/prometheus/prometheus.go @@ -36,7 +36,7 @@ func Handler(reg metrics.Registry) http.Handler { }) sort.Strings(names) - // Aggregate all the metris into a Prometheus collector + // Aggregate all the metrics into a Prometheus collector c := newCollector() for _, name := range names { diff --git a/metrics/registry_test.go b/metrics/registry_test.go index 6cfedfd88f005..d277ae5c3e477 100644 --- a/metrics/registry_test.go +++ b/metrics/registry_test.go @@ -307,5 +307,4 @@ func TestWalkRegistries(t *testing.T) { if prefix != "prefix.prefix2." { t.Fatal(prefix) } - } diff --git a/miner/miner.go b/miner/miner.go index 16c3bf19d263d..1e9607a76ad9d 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -40,7 +40,6 @@ import ( type Backend interface { BlockChain() *core.BlockChain TxPool() *core.TxPool - StateAtBlock(block *types.Block, reexec uint64, base *state.StateDB, checkLive bool, preferDisk bool) (statedb *state.StateDB, err error) } // Config is the configuration parameters of mining. diff --git a/miner/miner_test.go b/miner/miner_test.go index cf619845dd479..5bf344fd7076d 100644 --- a/miner/miner_test.go +++ b/miner/miner_test.go @@ -188,7 +188,6 @@ func TestStartStopMiner(t *testing.T) { waitForMiningState(t, miner, true) miner.Stop() waitForMiningState(t, miner, false) - } func TestCloseMiner(t *testing.T) { diff --git a/miner/stress/beacon/main.go b/miner/stress/beacon/main.go index 3f751049b89e6..88af84c7fcd3b 100644 --- a/miner/stress/beacon/main.go +++ b/miner/stress/beacon/main.go @@ -236,7 +236,7 @@ func newNodeManager(genesis *core.Genesis) *nodeManager { return &nodeManager{ close: make(chan struct{}), genesis: genesis, - genesisBlock: genesis.ToBlock(nil), + genesisBlock: genesis.ToBlock(), } } @@ -316,7 +316,7 @@ func (mgr *nodeManager) run() { nodes := mgr.getNodes(eth2MiningNode) nodes = append(nodes, mgr.getNodes(eth2NormalNode)...) nodes = append(nodes, mgr.getNodes(eth2LightClient)...) - for _, node := range append(nodes) { + for _, node := range nodes { fcState := beacon.ForkchoiceStateV1{ HeadBlockHash: oldest.Hash(), SafeBlockHash: common.Hash{}, diff --git a/miner/unconfirmed_test.go b/miner/unconfirmed_test.go index dc83cb92652d3..60958f658abce 100644 --- a/miner/unconfirmed_test.go +++ b/miner/unconfirmed_test.go @@ -74,7 +74,7 @@ func TestUnconfirmedShifts(t *testing.T) { if n := pool.blocks.Len(); n != int(limit)/2 { t.Errorf("unconfirmed count mismatch: have %d, want %d", n, limit/2) } - // Try to shift all the remaining blocks out and verify emptyness + // Try to shift all the remaining blocks out and verify emptiness pool.Shift(start + 2*uint64(limit)) if n := pool.blocks.Len(); n != 0 { t.Errorf("unconfirmed count mismatch: have %d, want %d", n, 0) diff --git a/miner/worker.go b/miner/worker.go index ae1b61d424111..93fb6288bb45c 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -756,16 +756,6 @@ func (w *worker) makeEnv(parent *types.Block, header *types.Header, coinbase com // Retrieve the parent state to execute on top and start a prefetcher for // the miner to speed block sealing up a bit. state, err := w.chain.StateAt(parent.Root()) - if err != nil { - // Note since the sealing block can be created upon the arbitrary parent - // block, but the state of parent block may already be pruned, so the necessary - // state recovery is needed here in the future. - // - // The maximum acceptable reorg depth can be limited by the finalised block - // somehow. TODO(rjl493456442) fix the hard-coded number here later. - state, err = w.eth.StateAtBlock(parent, 1024, nil, false, false) - log.Warn("Recovered mining state", "root", parent.Root(), "err", err) - } if err != nil { return nil, err } diff --git a/miner/worker_test.go b/miner/worker_test.go index 55361349bcca6..ec5ba67e1c65b 100644 --- a/miner/worker_test.go +++ b/miner/worker_test.go @@ -113,7 +113,6 @@ type testWorkerBackend struct { db ethdb.Database txPool *core.TxPool chain *core.BlockChain - testTxFeed event.Feed genesis *core.Genesis uncleBlock *types.Block } @@ -495,7 +494,7 @@ func testAdjustInterval(t *testing.T, chainConfig *params.ChainConfig, engine co } w.start() - time.Sleep(time.Second) // Ensure two tasks have been summitted due to start opt + time.Sleep(time.Second) // Ensure two tasks have been submitted due to start opt atomic.StoreUint32(&start, 1) w.setRecommitInterval(3 * time.Second) diff --git a/mobile/accounts.go b/mobile/accounts.go index 4d979bffff5da..d9eab93a741d0 100644 --- a/mobile/accounts.go +++ b/mobile/accounts.go @@ -212,10 +212,10 @@ func (ks *KeyStore) ImportECDSAKey(key []byte, passphrase string) (account *Acco // ImportPreSaleKey decrypts the given Ethereum presale wallet and stores // a key file in the key directory. The key file is encrypted with the same passphrase. -func (ks *KeyStore) ImportPreSaleKey(keyJSON []byte, passphrase string) (ccount *Account, _ error) { - account, err := ks.keystore.ImportPreSaleKey(common.CopyBytes(keyJSON), passphrase) +func (ks *KeyStore) ImportPreSaleKey(keyJSON []byte, passphrase string) (account *Account, _ error) { + acc, err := ks.keystore.ImportPreSaleKey(common.CopyBytes(keyJSON), passphrase) if err != nil { return nil, err } - return &Account{account}, nil + return &Account{acc}, nil } diff --git a/mobile/ethclient.go b/mobile/ethclient.go index 662125c4adeb0..00bcb3a2b9bc7 100644 --- a/mobile/ethclient.go +++ b/mobile/ethclient.go @@ -94,7 +94,6 @@ func (ec *EthereumClient) GetTransactionCount(ctx *Context, hash *Hash) (count i func (ec *EthereumClient) GetTransactionInBlock(ctx *Context, hash *Hash, index int) (tx *Transaction, _ error) { rawTx, err := ec.client.TransactionInBlock(ctx.context, hash.hash, uint(index)) return &Transaction{rawTx}, err - } // GetTransactionReceipt returns the receipt of a transaction by transaction hash. diff --git a/mobile/init.go b/mobile/init.go index 2025d85edc925..94f5baf28be71 100644 --- a/mobile/init.go +++ b/mobile/init.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -// Contains initialization code for the mbile library. +// Contains initialization code for the mobile library. package geth diff --git a/mobile/types.go b/mobile/types.go index a224f12ab23a2..f3f92e4d4ac30 100644 --- a/mobile/types.go +++ b/mobile/types.go @@ -55,7 +55,7 @@ func (n *Nonce) GetBytes() []byte { // GetHex retrieves the hex string representation of the block nonce. func (n *Nonce) GetHex() string { - return fmt.Sprintf("0x%x", n.nonce[:]) + return fmt.Sprintf("%#x", n.nonce[:]) } // String returns a printable representation of the nonce. @@ -75,7 +75,7 @@ func (b *Bloom) GetBytes() []byte { // GetHex retrieves the hex string representation of the bloom filter. func (b *Bloom) GetHex() string { - return fmt.Sprintf("0x%x", b.bloom[:]) + return fmt.Sprintf("%#x", b.bloom[:]) } // String returns a printable representation of the bloom filter. diff --git a/node/api.go b/node/api.go index 1b32399f635c7..67953a812e9dc 100644 --- a/node/api.go +++ b/node/api.go @@ -35,35 +35,26 @@ func (n *Node) apis() []rpc.API { return []rpc.API{ { Namespace: "admin", - Version: "1.0", - Service: &privateAdminAPI{n}, - }, { - Namespace: "admin", - Version: "1.0", - Service: &publicAdminAPI{n}, - Public: true, + Service: &adminAPI{n}, }, { Namespace: "debug", - Version: "1.0", Service: debug.Handler, }, { Namespace: "web3", - Version: "1.0", - Service: &publicWeb3API{n}, - Public: true, + Service: &web3API{n}, }, } } -// privateAdminAPI is the collection of administrative API methods exposed only -// over a secure RPC channel. -type privateAdminAPI struct { +// adminAPI is the collection of administrative API methods exposed over +// both secure and unsecure RPC channels. +type adminAPI struct { node *Node // Node interfaced by this API } // AddPeer requests connecting to a remote node, and also maintaining the new // connection at all times, even reconnecting if it is lost. -func (api *privateAdminAPI) AddPeer(url string) (bool, error) { +func (api *adminAPI) AddPeer(url string) (bool, error) { // Make sure the server is running, fail otherwise server := api.node.Server() if server == nil { @@ -79,7 +70,7 @@ func (api *privateAdminAPI) AddPeer(url string) (bool, error) { } // RemovePeer disconnects from a remote node if the connection exists -func (api *privateAdminAPI) RemovePeer(url string) (bool, error) { +func (api *adminAPI) RemovePeer(url string) (bool, error) { // Make sure the server is running, fail otherwise server := api.node.Server() if server == nil { @@ -95,7 +86,7 @@ func (api *privateAdminAPI) RemovePeer(url string) (bool, error) { } // AddTrustedPeer allows a remote node to always connect, even if slots are full -func (api *privateAdminAPI) AddTrustedPeer(url string) (bool, error) { +func (api *adminAPI) AddTrustedPeer(url string) (bool, error) { // Make sure the server is running, fail otherwise server := api.node.Server() if server == nil { @@ -111,7 +102,7 @@ func (api *privateAdminAPI) AddTrustedPeer(url string) (bool, error) { // RemoveTrustedPeer removes a remote node from the trusted peer set, but it // does not disconnect it automatically. -func (api *privateAdminAPI) RemoveTrustedPeer(url string) (bool, error) { +func (api *adminAPI) RemoveTrustedPeer(url string) (bool, error) { // Make sure the server is running, fail otherwise server := api.node.Server() if server == nil { @@ -127,7 +118,7 @@ func (api *privateAdminAPI) RemoveTrustedPeer(url string) (bool, error) { // PeerEvents creates an RPC subscription which receives peer events from the // node's p2p.Server -func (api *privateAdminAPI) PeerEvents(ctx context.Context) (*rpc.Subscription, error) { +func (api *adminAPI) PeerEvents(ctx context.Context) (*rpc.Subscription, error) { // Make sure the server is running, fail otherwise server := api.node.Server() if server == nil { @@ -164,7 +155,7 @@ func (api *privateAdminAPI) PeerEvents(ctx context.Context) (*rpc.Subscription, } // StartHTTP starts the HTTP RPC API server. -func (api *privateAdminAPI) StartHTTP(host *string, port *int, cors *string, apis *string, vhosts *string) (bool, error) { +func (api *adminAPI) StartHTTP(host *string, port *int, cors *string, apis *string, vhosts *string) (bool, error) { api.node.lock.Lock() defer api.node.lock.Unlock() @@ -219,26 +210,26 @@ func (api *privateAdminAPI) StartHTTP(host *string, port *int, cors *string, api // StartRPC starts the HTTP RPC API server. // Deprecated: use StartHTTP instead. -func (api *privateAdminAPI) StartRPC(host *string, port *int, cors *string, apis *string, vhosts *string) (bool, error) { +func (api *adminAPI) StartRPC(host *string, port *int, cors *string, apis *string, vhosts *string) (bool, error) { log.Warn("Deprecation warning", "method", "admin.StartRPC", "use-instead", "admin.StartHTTP") return api.StartHTTP(host, port, cors, apis, vhosts) } // StopHTTP shuts down the HTTP server. -func (api *privateAdminAPI) StopHTTP() (bool, error) { +func (api *adminAPI) StopHTTP() (bool, error) { api.node.http.stop() return true, nil } // StopRPC shuts down the HTTP server. // Deprecated: use StopHTTP instead. -func (api *privateAdminAPI) StopRPC() (bool, error) { +func (api *adminAPI) StopRPC() (bool, error) { log.Warn("Deprecation warning", "method", "admin.StopRPC", "use-instead", "admin.StopHTTP") return api.StopHTTP() } // StartWS starts the websocket RPC API server. -func (api *privateAdminAPI) StartWS(host *string, port *int, allowedOrigins *string, apis *string) (bool, error) { +func (api *adminAPI) StartWS(host *string, port *int, allowedOrigins *string, apis *string) (bool, error) { api.node.lock.Lock() defer api.node.lock.Unlock() @@ -290,21 +281,15 @@ func (api *privateAdminAPI) StartWS(host *string, port *int, allowedOrigins *str } // StopWS terminates all WebSocket servers. -func (api *privateAdminAPI) StopWS() (bool, error) { +func (api *adminAPI) StopWS() (bool, error) { api.node.http.stopWS() api.node.ws.stop() return true, nil } -// publicAdminAPI is the collection of administrative API methods exposed over -// both secure and unsecure RPC channels. -type publicAdminAPI struct { - node *Node // Node interfaced by this API -} - // Peers retrieves all the information we know about each individual peer at the // protocol granularity. -func (api *publicAdminAPI) Peers() ([]*p2p.PeerInfo, error) { +func (api *adminAPI) Peers() ([]*p2p.PeerInfo, error) { server := api.node.Server() if server == nil { return nil, ErrNodeStopped @@ -314,7 +299,7 @@ func (api *publicAdminAPI) Peers() ([]*p2p.PeerInfo, error) { // NodeInfo retrieves all the information we know about the host node at the // protocol granularity. -func (api *publicAdminAPI) NodeInfo() (*p2p.NodeInfo, error) { +func (api *adminAPI) NodeInfo() (*p2p.NodeInfo, error) { server := api.node.Server() if server == nil { return nil, ErrNodeStopped @@ -323,22 +308,22 @@ func (api *publicAdminAPI) NodeInfo() (*p2p.NodeInfo, error) { } // Datadir retrieves the current data directory the node is using. -func (api *publicAdminAPI) Datadir() string { +func (api *adminAPI) Datadir() string { return api.node.DataDir() } -// publicWeb3API offers helper utils -type publicWeb3API struct { +// web3API offers helper utils +type web3API struct { stack *Node } // ClientVersion returns the node name -func (s *publicWeb3API) ClientVersion() string { +func (s *web3API) ClientVersion() string { return s.stack.Server().Name } // Sha3 applies the ethereum sha3 implementation on the input. // It assumes the input is hex encoded. -func (s *publicWeb3API) Sha3(input hexutil.Bytes) hexutil.Bytes { +func (s *web3API) Sha3(input hexutil.Bytes) hexutil.Bytes { return crypto.Keccak256(input) } diff --git a/node/api_test.go b/node/api_test.go index 9549adf9c254b..d76cb943e4ee9 100644 --- a/node/api_test.go +++ b/node/api_test.go @@ -35,7 +35,7 @@ func TestStartRPC(t *testing.T) { type test struct { name string cfg Config - fn func(*testing.T, *Node, *privateAdminAPI) + fn func(*testing.T, *Node, *adminAPI) // Checks. These run after the node is configured and all API calls have been made. wantReachable bool // whether the HTTP server should be reachable at all @@ -48,7 +48,7 @@ func TestStartRPC(t *testing.T) { { name: "all off", cfg: Config{}, - fn: func(t *testing.T, n *Node, api *privateAdminAPI) { + fn: func(t *testing.T, n *Node, api *adminAPI) { }, wantReachable: false, wantHandlers: false, @@ -58,7 +58,7 @@ func TestStartRPC(t *testing.T) { { name: "rpc enabled through config", cfg: Config{HTTPHost: "127.0.0.1"}, - fn: func(t *testing.T, n *Node, api *privateAdminAPI) { + fn: func(t *testing.T, n *Node, api *adminAPI) { }, wantReachable: true, wantHandlers: true, @@ -68,7 +68,7 @@ func TestStartRPC(t *testing.T) { { name: "rpc enabled through API", cfg: Config{}, - fn: func(t *testing.T, n *Node, api *privateAdminAPI) { + fn: func(t *testing.T, n *Node, api *adminAPI) { _, err := api.StartHTTP(sp("127.0.0.1"), ip(0), nil, nil, nil) assert.NoError(t, err) }, @@ -80,7 +80,7 @@ func TestStartRPC(t *testing.T) { { name: "rpc start again after failure", cfg: Config{}, - fn: func(t *testing.T, n *Node, api *privateAdminAPI) { + fn: func(t *testing.T, n *Node, api *adminAPI) { // Listen on a random port. listener, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { @@ -108,7 +108,7 @@ func TestStartRPC(t *testing.T) { { name: "rpc stopped through API", cfg: Config{HTTPHost: "127.0.0.1"}, - fn: func(t *testing.T, n *Node, api *privateAdminAPI) { + fn: func(t *testing.T, n *Node, api *adminAPI) { _, err := api.StopHTTP() assert.NoError(t, err) }, @@ -120,7 +120,7 @@ func TestStartRPC(t *testing.T) { { name: "rpc stopped twice", cfg: Config{HTTPHost: "127.0.0.1"}, - fn: func(t *testing.T, n *Node, api *privateAdminAPI) { + fn: func(t *testing.T, n *Node, api *adminAPI) { _, err := api.StopHTTP() assert.NoError(t, err) @@ -143,7 +143,7 @@ func TestStartRPC(t *testing.T) { { name: "ws enabled through API", cfg: Config{}, - fn: func(t *testing.T, n *Node, api *privateAdminAPI) { + fn: func(t *testing.T, n *Node, api *adminAPI) { _, err := api.StartWS(sp("127.0.0.1"), ip(0), nil, nil) assert.NoError(t, err) }, @@ -155,7 +155,7 @@ func TestStartRPC(t *testing.T) { { name: "ws stopped through API", cfg: Config{WSHost: "127.0.0.1"}, - fn: func(t *testing.T, n *Node, api *privateAdminAPI) { + fn: func(t *testing.T, n *Node, api *adminAPI) { _, err := api.StopWS() assert.NoError(t, err) }, @@ -167,7 +167,7 @@ func TestStartRPC(t *testing.T) { { name: "ws stopped twice", cfg: Config{WSHost: "127.0.0.1"}, - fn: func(t *testing.T, n *Node, api *privateAdminAPI) { + fn: func(t *testing.T, n *Node, api *adminAPI) { _, err := api.StopWS() assert.NoError(t, err) @@ -182,7 +182,7 @@ func TestStartRPC(t *testing.T) { { name: "ws enabled after RPC", cfg: Config{HTTPHost: "127.0.0.1"}, - fn: func(t *testing.T, n *Node, api *privateAdminAPI) { + fn: func(t *testing.T, n *Node, api *adminAPI) { wsport := n.http.port _, err := api.StartWS(sp("127.0.0.1"), ip(wsport), nil, nil) assert.NoError(t, err) @@ -195,7 +195,7 @@ func TestStartRPC(t *testing.T) { { name: "ws enabled after RPC then stopped", cfg: Config{HTTPHost: "127.0.0.1"}, - fn: func(t *testing.T, n *Node, api *privateAdminAPI) { + fn: func(t *testing.T, n *Node, api *adminAPI) { wsport := n.http.port _, err := api.StartWS(sp("127.0.0.1"), ip(wsport), nil, nil) assert.NoError(t, err) @@ -210,7 +210,7 @@ func TestStartRPC(t *testing.T) { }, { name: "rpc stopped with ws enabled", - fn: func(t *testing.T, n *Node, api *privateAdminAPI) { + fn: func(t *testing.T, n *Node, api *adminAPI) { _, err := api.StartHTTP(sp("127.0.0.1"), ip(0), nil, nil, nil) assert.NoError(t, err) @@ -228,7 +228,7 @@ func TestStartRPC(t *testing.T) { }, { name: "rpc enabled after ws", - fn: func(t *testing.T, n *Node, api *privateAdminAPI) { + fn: func(t *testing.T, n *Node, api *adminAPI) { _, err := api.StartWS(sp("127.0.0.1"), ip(0), nil, nil) assert.NoError(t, err) @@ -271,7 +271,7 @@ func TestStartRPC(t *testing.T) { // Run the API call hook. if test.fn != nil { - test.fn(t, stack, &privateAdminAPI{stack}) + test.fn(t, stack, &adminAPI{stack}) } // Check if the HTTP endpoints are available. diff --git a/node/config_test.go b/node/config_test.go index d9f812ec4cb2c..e8af8ddcd87b4 100644 --- a/node/config_test.go +++ b/node/config_test.go @@ -63,12 +63,9 @@ func TestDatadirCreation(t *testing.T) { }() dir = filepath.Join(file.Name(), "invalid/path") - node, err = New(&Config{DataDir: dir}) + _, err = New(&Config{DataDir: dir}) if err == nil { t.Fatalf("protocol stack created with an invalid datadir") - if err := node.Close(); err != nil { - t.Fatalf("failed to close node: %v", err) - } } } @@ -118,7 +115,7 @@ func TestNodeKeyPersistency(t *testing.T) { } config := &Config{Name: "unit-test", DataDir: dir, P2P: p2p.Config{PrivateKey: key}} config.NodeKey() - if _, err := os.Stat(filepath.Join(keyfile)); err == nil { + if _, err := os.Stat(keyfile); err == nil { t.Fatalf("one-shot node key persisted to data directory") } @@ -139,7 +136,7 @@ func TestNodeKeyPersistency(t *testing.T) { // Configure a new node and ensure the previously persisted key is loaded config = &Config{Name: "unit-test", DataDir: dir} config.NodeKey() - blob2, err := os.ReadFile(filepath.Join(keyfile)) + blob2, err := os.ReadFile(keyfile) if err != nil { t.Fatalf("failed to read previously persisted node key: %v", err) } diff --git a/node/endpoints.go b/node/endpoints.go index efc311e7e3173..14c12fd1f175b 100644 --- a/node/endpoints.go +++ b/node/endpoints.go @@ -39,10 +39,11 @@ func StartHTTPEndpoint(endpoint string, timeouts rpc.HTTPTimeouts, handler http. CheckTimeouts(&timeouts) // Bundle and start the HTTP server httpSrv := &http.Server{ - Handler: handler, - ReadTimeout: timeouts.ReadTimeout, - WriteTimeout: timeouts.WriteTimeout, - IdleTimeout: timeouts.IdleTimeout, + Handler: handler, + ReadTimeout: timeouts.ReadTimeout, + ReadHeaderTimeout: timeouts.ReadHeaderTimeout, + WriteTimeout: timeouts.WriteTimeout, + IdleTimeout: timeouts.IdleTimeout, } go httpSrv.Serve(listener) return httpSrv, listener.Addr(), err @@ -75,6 +76,10 @@ func CheckTimeouts(timeouts *rpc.HTTPTimeouts) { log.Warn("Sanitizing invalid HTTP read timeout", "provided", timeouts.ReadTimeout, "updated", rpc.DefaultHTTPTimeouts.ReadTimeout) timeouts.ReadTimeout = rpc.DefaultHTTPTimeouts.ReadTimeout } + if timeouts.ReadHeaderTimeout < time.Second { + log.Warn("Sanitizing invalid HTTP read header timeout", "provided", timeouts.ReadHeaderTimeout, "updated", rpc.DefaultHTTPTimeouts.ReadHeaderTimeout) + timeouts.ReadHeaderTimeout = rpc.DefaultHTTPTimeouts.ReadHeaderTimeout + } if timeouts.WriteTimeout < time.Second { log.Warn("Sanitizing invalid HTTP write timeout", "provided", timeouts.WriteTimeout, "updated", rpc.DefaultHTTPTimeouts.WriteTimeout) timeouts.WriteTimeout = rpc.DefaultHTTPTimeouts.WriteTimeout diff --git a/node/jwt_handler.go b/node/jwt_handler.go index 28d5b87c60bcc..363f6b3aad47e 100644 --- a/node/jwt_handler.go +++ b/node/jwt_handler.go @@ -24,6 +24,8 @@ import ( "github.com/golang-jwt/jwt/v4" ) +const jwtExpiryTimeout = 60 * time.Second + type jwtHandler struct { keyFunc func(token *jwt.Token) (interface{}, error) next http.Handler @@ -68,9 +70,9 @@ func (handler *jwtHandler) ServeHTTP(out http.ResponseWriter, r *http.Request) { http.Error(out, "token is expired", http.StatusForbidden) case claims.IssuedAt == nil: http.Error(out, "missing issued-at", http.StatusForbidden) - case time.Since(claims.IssuedAt.Time) > 5*time.Second: + case time.Since(claims.IssuedAt.Time) > jwtExpiryTimeout: http.Error(out, "stale token", http.StatusForbidden) - case time.Until(claims.IssuedAt.Time) > 5*time.Second: + case time.Until(claims.IssuedAt.Time) > jwtExpiryTimeout: http.Error(out, "future token", http.StatusForbidden) default: handler.next.ServeHTTP(out, r) diff --git a/node/node.go b/node/node.go index 7c540306db2b1..b60e32f22fd2c 100644 --- a/node/node.go +++ b/node/node.go @@ -20,6 +20,7 @@ import ( crand "crypto/rand" "errors" "fmt" + "hash/crc32" "net/http" "os" "path/filepath" @@ -352,10 +353,10 @@ func (n *Node) obtainJWTSecret(cliParam string) ([]byte, error) { fileName = n.ResolvePath(datadirJWTKey) } // try reading from file - log.Debug("Reading JWT secret", "path", fileName) if data, err := os.ReadFile(fileName); err == nil { jwtSecret := common.FromHex(strings.TrimSpace(string(data))) if len(jwtSecret) == 32 { + log.Info("Loaded JWT secret file", "path", fileName, "crc32", fmt.Sprintf("%#x", crc32.ChecksumIEEE(jwtSecret))) return jwtSecret, nil } log.Error("Invalid JWT secret", "path", fileName, "length", len(jwtSecret)) @@ -702,7 +703,7 @@ func (n *Node) OpenDatabase(name string, cache, handles int, namespace string, r // also attaching a chain freezer to it that moves ancient chain data from the // database to immutable append-only files. If the node is an ephemeral one, a // memory database is returned. -func (n *Node) OpenDatabaseWithFreezer(name string, cache, handles int, freezer, namespace string, readonly bool) (ethdb.Database, error) { +func (n *Node) OpenDatabaseWithFreezer(name string, cache, handles int, ancient string, namespace string, readonly bool) (ethdb.Database, error) { n.lock.Lock() defer n.lock.Unlock() if n.state == closedState { @@ -714,14 +715,7 @@ func (n *Node) OpenDatabaseWithFreezer(name string, cache, handles int, freezer, if n.config.DataDir == "" { db = rawdb.NewMemoryDatabase() } else { - root := n.ResolvePath(name) - switch { - case freezer == "": - freezer = filepath.Join(root, "ancient") - case !filepath.IsAbs(freezer): - freezer = n.ResolvePath(freezer) - } - db, err = rawdb.NewLevelDBDatabaseWithFreezer(root, cache, handles, freezer, namespace, readonly) + db, err = rawdb.NewLevelDBDatabaseWithFreezer(n.ResolvePath(name), cache, handles, n.ResolveAncient(name, ancient), namespace, readonly) } if err == nil { @@ -735,6 +729,17 @@ func (n *Node) ResolvePath(x string) string { return n.config.ResolvePath(x) } +// ResolveAncient returns the absolute path of the root ancient directory. +func (n *Node) ResolveAncient(name string, ancient string) string { + switch { + case ancient == "": + ancient = filepath.Join(n.ResolvePath(name), "ancient") + case !filepath.IsAbs(ancient): + ancient = n.ResolvePath(ancient) + } + return ancient +} + // closeTrackingDB wraps the Close method of a database. When the database is closed by the // service, the wrapper removes it from the node's database map. This ensures that Node // won't auto-close the database if it is closed by the service that opened it. diff --git a/node/node_test.go b/node/node_test.go index 9f9febcacbfee..7c76e21f6baf0 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -581,7 +581,6 @@ func (test rpcPrefixTest) check(t *testing.T, node *Node) { if err == nil { t.Errorf("Error: %s: WebSocket connection succeeded for path in wantNoWS", path) } - } } @@ -614,7 +613,6 @@ func doHTTPRequest(t *testing.T, req *http.Request) *http.Response { resp, err := client.Do(req) if err != nil { t.Fatalf("could not issue a GET request to the given endpoint: %v", err) - } return resp } diff --git a/node/rpcstack.go b/node/rpcstack.go index 0d2be9008a41e..5d411fa61e81f 100644 --- a/node/rpcstack.go +++ b/node/rpcstack.go @@ -27,6 +27,7 @@ import ( "strings" "sync" "sync/atomic" + "time" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rpc" @@ -81,6 +82,10 @@ type httpServer struct { handlerNames map[string]string } +const ( + shutdownTimeout = 5 * time.Second +) + func newHTTPServer(log log.Logger, timeouts rpc.HTTPTimeouts) *httpServer { h := &httpServer{log: log, timeouts: timeouts, handlerNames: make(map[string]string)} @@ -129,6 +134,7 @@ func (h *httpServer) start() error { if h.timeouts != (rpc.HTTPTimeouts{}) { CheckTimeouts(&h.timeouts) h.server.ReadTimeout = h.timeouts.ReadTimeout + h.server.ReadHeaderTimeout = h.timeouts.ReadHeaderTimeout h.server.WriteTimeout = h.timeouts.WriteTimeout h.server.IdleTimeout = h.timeouts.IdleTimeout } @@ -261,7 +267,13 @@ func (h *httpServer) doStop() { h.wsHandler.Store((*rpcHandler)(nil)) wsHandler.server.Stop() } - h.server.Shutdown(context.Background()) + ctx, cancel := context.WithTimeout(context.Background(), shutdownTimeout) + defer cancel() + err := h.server.Shutdown(ctx) + if err == ctx.Err() { + h.log.Warn("HTTP server graceful shutdown timed out") + h.server.Close() + } h.listener.Close() h.log.Info("HTTP server stopped", "endpoint", h.listener.Addr()) @@ -281,7 +293,7 @@ func (h *httpServer) enableRPC(apis []rpc.API, config httpConfig) error { // Create RPC server and handler. srv := rpc.NewServer() - if err := RegisterApis(apis, config.Modules, srv, false); err != nil { + if err := RegisterApis(apis, config.Modules, srv); err != nil { return err } h.httpConfig = config @@ -312,7 +324,7 @@ func (h *httpServer) enableWS(apis []rpc.API, config wsConfig) error { } // Create RPC server and handler. srv := rpc.NewServer() - if err := RegisterApis(apis, config.Modules, srv, false); err != nil { + if err := RegisterApis(apis, config.Modules, srv); err != nil { return err } h.wsConfig = config @@ -427,7 +439,6 @@ func (h *virtualHostHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { // It's an IP address, we can serve that h.next.ServeHTTP(w, r) return - } // Not an IP address, but a hostname. Need to validate if _, exist := h.vhosts["*"]; exist { @@ -528,7 +539,7 @@ func (is *ipcServer) stop() error { // RegisterApis checks the given modules' availability, generates an allowlist based on the allowed modules, // and then registers all of the APIs exposed by the services. -func RegisterApis(apis []rpc.API, modules []string, srv *rpc.Server, exposeAll bool) error { +func RegisterApis(apis []rpc.API, modules []string, srv *rpc.Server) error { if bad, available := checkModuleAvailability(modules, apis); len(bad) > 0 { log.Error("Unavailable modules in HTTP API list", "unavailable", bad, "available", available) } @@ -539,7 +550,7 @@ func RegisterApis(apis []rpc.API, modules []string, srv *rpc.Server, exposeAll b } // Register all the APIs exposed by the services for _, api := range apis { - if exposeAll || allowList[api.Namespace] || (len(allowList) == 0 && api.Public) { + if allowList[api.Namespace] || len(allowList) == 0 { if err := srv.RegisterName(api.Namespace, api.Service); err != nil { return err } diff --git a/node/rpcstack_test.go b/node/rpcstack_test.go index c7dba8a1ef0d1..09acf7ea04583 100644 --- a/node/rpcstack_test.go +++ b/node/rpcstack_test.go @@ -100,7 +100,7 @@ func TestWebsocketOrigins(t *testing.T) { expFail: []string{ "test", // no scheme, required by spec "http://test", // wrong scheme - "http://test.foo", "https://a.test.x", // subdomain variatoins + "http://test.foo", "https://a.test.x", // subdomain variations "http://testx:8540", "https://xtest:8540"}, }, // ip tests @@ -314,60 +314,105 @@ func TestJWT(t *testing.T) { ss, _ := jwt.NewWithClaims(method, testClaim(input)).SignedString(secret) return ss } - expOk := []string{ - fmt.Sprintf("Bearer %v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix()})), - fmt.Sprintf("Bearer %v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix() + 4})), - fmt.Sprintf("Bearer %v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix() - 4})), - fmt.Sprintf("Bearer %v", issueToken(secret, nil, testClaim{ - "iat": time.Now().Unix(), - "exp": time.Now().Unix() + 2, - })), - fmt.Sprintf("Bearer %v", issueToken(secret, nil, testClaim{ - "iat": time.Now().Unix(), - "bar": "baz", - })), - } - expFail := []string{ - // future - fmt.Sprintf("Bearer %v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix() + 6})), - // stale - fmt.Sprintf("Bearer %v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix() - 6})), - // wrong algo - fmt.Sprintf("Bearer %v", issueToken(secret, jwt.SigningMethodHS512, testClaim{"iat": time.Now().Unix() + 4})), - // expired - fmt.Sprintf("Bearer %v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix(), "exp": time.Now().Unix()})), - // missing mandatory iat - fmt.Sprintf("Bearer %v", issueToken(secret, nil, testClaim{})), - // wrong secret - fmt.Sprintf("Bearer %v", issueToken([]byte("wrong"), nil, testClaim{"iat": time.Now().Unix()})), - fmt.Sprintf("Bearer %v", issueToken([]byte{}, nil, testClaim{"iat": time.Now().Unix()})), - fmt.Sprintf("Bearer %v", issueToken(nil, nil, testClaim{"iat": time.Now().Unix()})), - // Various malformed syntax - fmt.Sprintf("%v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix()})), - fmt.Sprintf("Bearer %v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix()})), - fmt.Sprintf("bearer %v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix()})), - fmt.Sprintf("Bearer: %v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix()})), - fmt.Sprintf("Bearer:%v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix()})), - fmt.Sprintf("Bearer\t%v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix()})), - fmt.Sprintf("Bearer \t%v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix()})), - } srv := createAndStartServer(t, &httpConfig{jwtSecret: []byte("secret")}, true, &wsConfig{Origins: []string{"*"}, jwtSecret: []byte("secret")}) wsUrl := fmt.Sprintf("ws://%v", srv.listenAddr()) htUrl := fmt.Sprintf("http://%v", srv.listenAddr()) - for i, token := range expOk { + expOk := []func() string{ + func() string { + return fmt.Sprintf("Bearer %v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix()})) + }, + func() string { + return fmt.Sprintf("Bearer %v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix() + 4})) + }, + func() string { + return fmt.Sprintf("Bearer %v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix() - 4})) + }, + func() string { + return fmt.Sprintf("Bearer %v", issueToken(secret, nil, testClaim{ + "iat": time.Now().Unix(), + "exp": time.Now().Unix() + 2, + })) + }, + func() string { + return fmt.Sprintf("Bearer %v", issueToken(secret, nil, testClaim{ + "iat": time.Now().Unix(), + "bar": "baz", + })) + }, + } + for i, tokenFn := range expOk { + token := tokenFn() if err := wsRequest(t, wsUrl, "Authorization", token); err != nil { t.Errorf("test %d-ws, token '%v': expected ok, got %v", i, token, err) } + token = tokenFn() if resp := rpcRequest(t, htUrl, "Authorization", token); resp.StatusCode != 200 { t.Errorf("test %d-http, token '%v': expected ok, got %v", i, token, resp.StatusCode) } } - for i, token := range expFail { + + expFail := []func() string{ + // future + func() string { + return fmt.Sprintf("Bearer %v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix() + int64(jwtExpiryTimeout.Seconds()) + 1})) + }, + // stale + func() string { + return fmt.Sprintf("Bearer %v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix() - int64(jwtExpiryTimeout.Seconds()) - 1})) + }, + // wrong algo + func() string { + return fmt.Sprintf("Bearer %v", issueToken(secret, jwt.SigningMethodHS512, testClaim{"iat": time.Now().Unix() + 4})) + }, + // expired + func() string { + return fmt.Sprintf("Bearer %v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix(), "exp": time.Now().Unix()})) + }, + // missing mandatory iat + func() string { + return fmt.Sprintf("Bearer %v", issueToken(secret, nil, testClaim{})) + }, + // wrong secret + func() string { + return fmt.Sprintf("Bearer %v", issueToken([]byte("wrong"), nil, testClaim{"iat": time.Now().Unix()})) + }, + func() string { + return fmt.Sprintf("Bearer %v", issueToken([]byte{}, nil, testClaim{"iat": time.Now().Unix()})) + }, + func() string { + return fmt.Sprintf("Bearer %v", issueToken(nil, nil, testClaim{"iat": time.Now().Unix()})) + }, + // Various malformed syntax + func() string { + return fmt.Sprintf("%v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix()})) + }, + func() string { + return fmt.Sprintf("Bearer %v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix()})) + }, + func() string { + return fmt.Sprintf("bearer %v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix()})) + }, + func() string { + return fmt.Sprintf("Bearer: %v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix()})) + }, + func() string { + return fmt.Sprintf("Bearer:%v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix()})) + }, + func() string { + return fmt.Sprintf("Bearer\t%v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix()})) + }, + func() string { + return fmt.Sprintf("Bearer \t%v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix()})) + }, + } + for i, tokenFn := range expFail { + token := tokenFn() if err := wsRequest(t, wsUrl, "Authorization", token); err == nil { t.Errorf("tc %d-ws, token '%v': expected not to allow, got ok", i, token) } + token = tokenFn() if resp := rpcRequest(t, htUrl, "Authorization", token); resp.StatusCode != 403 { t.Errorf("tc %d-http, token '%v': expected not to allow, got %v", i, token, resp.StatusCode) } diff --git a/node/utils_test.go b/node/utils_test.go index b7474bb706180..681f3a8b285c4 100644 --- a/node/utils_test.go +++ b/node/utils_test.go @@ -47,8 +47,6 @@ type InstrumentedService struct { startHook func() stopHook func() - - protocols []p2p.Protocol } func (s *InstrumentedService) Start() error { @@ -97,17 +95,12 @@ func (f *FullService) APIs() []rpc.API { return []rpc.API{ { Namespace: "admin", - Version: "1.0", }, { Namespace: "debug", - Version: "1.0", - Public: true, }, { Namespace: "net", - Version: "1.0", - Public: true, }, } } diff --git a/p2p/discover/lookup.go b/p2p/discover/lookup.go index 9ab4a71ce7b42..b8d97b44e1cc5 100644 --- a/p2p/discover/lookup.go +++ b/p2p/discover/lookup.go @@ -18,6 +18,7 @@ package discover import ( "context" + "errors" "time" "github.com/ethereum/go-ethereum/p2p/enode" @@ -141,7 +142,7 @@ func (it *lookup) slowdown() { func (it *lookup) query(n *node, reply chan<- []*node) { fails := it.tab.db.FindFails(n.ID(), n.IP()) r, err := it.queryfunc(n) - if err == errClosed { + if errors.Is(err, errClosed) { // Avoid recording failures on shutdown. reply <- nil return diff --git a/p2p/discover/ntp.go b/p2p/discover/ntp.go index 1bb52399fbc54..48ceffe95b8db 100644 --- a/p2p/discover/ntp.go +++ b/p2p/discover/ntp.go @@ -108,7 +108,7 @@ func sntpDrift(measurements int) (time.Duration, error) { // Calculate the drift based on an assumed answer time of RRT/2 drifts = append(drifts, sent.Sub(t)+elapsed/2) } - // Calculate average drif (drop two extremities to avoid outliers) + // Calculate average drift (drop two extremities to avoid outliers) sort.Sort(durationSlice(drifts)) drift := time.Duration(0) diff --git a/p2p/discover/table_util_test.go b/p2p/discover/table_util_test.go index 47a2e7ac3caf1..77e03ca9e7e46 100644 --- a/p2p/discover/table_util_test.go +++ b/p2p/discover/table_util_test.go @@ -134,8 +134,8 @@ func newPingRecorder() *pingRecorder { } } -// setRecord updates a node record. Future calls to ping and -// requestENR will return this record. +// updateRecord updates a node record. Future calls to ping and +// RequestENR will return this record. func (t *pingRecorder) updateRecord(n *enode.Node) { t.mu.Lock() defer t.mu.Unlock() @@ -162,7 +162,7 @@ func (t *pingRecorder) ping(n *enode.Node) (seq uint64, err error) { return seq, nil } -// requestENR simulates an ENR request. +// RequestENR simulates an ENR request. func (t *pingRecorder) RequestENR(n *enode.Node) (*enode.Node, error) { t.mu.Lock() defer t.mu.Unlock() diff --git a/p2p/discover/v4_udp.go b/p2p/discover/v4_udp.go index 334716aebed01..67cd2c004cf6e 100644 --- a/p2p/discover/v4_udp.go +++ b/p2p/discover/v4_udp.go @@ -328,13 +328,13 @@ func (t *UDPv4) findnode(toid enode.ID, toaddr *net.UDPAddr, target v4wire.Pubke // enough nodes the reply matcher will time out waiting for the second reply, but // there's no need for an error in that case. err := <-rm.errc - if err == errTimeout && rm.reply != nil { + if errors.Is(err, errTimeout) && rm.reply != nil { err = nil } return nodes, err } -// RequestENR sends enrRequest to the given node and waits for a response. +// RequestENR sends ENRRequest to the given node and waits for a response. func (t *UDPv4) RequestENR(n *enode.Node) (*enode.Node, error) { addr := &net.UDPAddr{IP: n.IP(), Port: n.UDP()} t.ensureBond(n.ID(), addr) @@ -525,8 +525,8 @@ func (t *UDPv4) readLoop(unhandled chan<- ReadPacket) { t.log.Debug("Temporary UDP read error", "err", err) continue } else if err != nil { - // Shut down the loop for permament errors. - if err != io.EOF { + // Shut down the loop for permanent errors. + if !errors.Is(err, io.EOF) { t.log.Debug("UDP read error", "err", err) } return diff --git a/p2p/discover/v4_udp_test.go b/p2p/discover/v4_udp_test.go index e36912f010aeb..e00bf2784cb06 100644 --- a/p2p/discover/v4_udp_test.go +++ b/p2p/discover/v4_udp_test.go @@ -284,6 +284,7 @@ func TestUDPv4_findnode(t *testing.T) { test.waitPacketOut(func(p *v4wire.Neighbors, to *net.UDPAddr, hash []byte) { if len(p.Nodes) != len(want) { t.Errorf("wrong number of results: got %d, want %d", len(p.Nodes), bucketSize) + return } for i, n := range p.Nodes { if n.ID.ID() != want[i].ID() { @@ -489,7 +490,7 @@ func TestUDPv4_EIP868(t *testing.T) { t.Fatalf("invalid record: %v", err) } if !reflect.DeepEqual(n, wantNode) { - t.Fatalf("wrong node in enrResponse: %v", n) + t.Fatalf("wrong node in ENRResponse: %v", n) } }) } diff --git a/p2p/discover/v4wire/v4wire.go b/p2p/discover/v4wire/v4wire.go index d6bf3dc4600af..b07a6e341c31d 100644 --- a/p2p/discover/v4wire/v4wire.go +++ b/p2p/discover/v4wire/v4wire.go @@ -86,16 +86,16 @@ type ( Rest []rlp.RawValue `rlp:"tail"` } - // enrRequest queries for the remote node's record. + // ENRRequest queries for the remote node's record. ENRRequest struct { Expiration uint64 // Ignore additional fields (for forward compatibility). Rest []rlp.RawValue `rlp:"tail"` } - // enrResponse is the reply to enrRequest. + // ENRResponse is the reply to ENRRequest. ENRResponse struct { - ReplyTok []byte // Hash of the enrRequest packet. + ReplyTok []byte // Hash of the ENRRequest packet. Record enr.Record // Ignore additional fields (for forward compatibility). Rest []rlp.RawValue `rlp:"tail"` diff --git a/p2p/discover/v5_udp.go b/p2p/discover/v5_udp.go index dc63382fc9011..071ed65adc7f5 100644 --- a/p2p/discover/v5_udp.go +++ b/p2p/discover/v5_udp.go @@ -305,7 +305,7 @@ func (t *UDPv5) lookupWorker(destNode *node, target enode.ID) ([]*node, error) { ) var r []*enode.Node r, err = t.findnode(unwrapNode(destNode), dists) - if err == errClosed { + if errors.Is(err, errClosed) { return nil, err } for _, n := range r { @@ -347,7 +347,7 @@ func (t *UDPv5) ping(n *enode.Node) (uint64, error) { } } -// requestENR requests n's record. +// RequestENR requests n's record. func (t *UDPv5) RequestENR(n *enode.Node) (*enode.Node, error) { nodes, err := t.findnode(n, []uint{0}) if err != nil { @@ -407,6 +407,9 @@ func (t *UDPv5) verifyResponseNode(c *callV5, r *enr.Record, distances []uint, s if err := netutil.CheckRelayIP(c.node.IP(), node.IP()); err != nil { return nil, err } + if t.netrestrict != nil && !t.netrestrict.Contains(node.IP()) { + return nil, errors.New("not contained in netrestrict list") + } if c.node.UDP() <= 1024 { return nil, errLowPort } @@ -622,8 +625,8 @@ func (t *UDPv5) readLoop() { t.log.Debug("Temporary UDP read error", "err", err) continue } else if err != nil { - // Shut down the loop for permament errors. - if err != io.EOF { + // Shut down the loop for permanent errors. + if !errors.Is(err, io.EOF) { t.log.Debug("UDP read error", "err", err) } return diff --git a/p2p/discover/v5wire/encoding.go b/p2p/discover/v5wire/encoding.go index 7d17281ef9691..45f2f0883bad8 100644 --- a/p2p/discover/v5wire/encoding.go +++ b/p2p/discover/v5wire/encoding.go @@ -300,7 +300,7 @@ func (c *Codec) encodeWhoareyou(toID enode.ID, packet *Whoareyou) (Header, error return head, nil } -// encodeHandshakeMessage encodes the handshake message packet header. +// encodeHandshakeHeader encodes the handshake message packet header. func (c *Codec) encodeHandshakeHeader(toID enode.ID, addr string, challenge *Whoareyou) (Header, *session, error) { // Ensure calling code sets challenge.node. if challenge.Node == nil { @@ -337,7 +337,7 @@ func (c *Codec) encodeHandshakeHeader(toID enode.ID, addr string, challenge *Who return head, session, err } -// encodeAuthHeader creates the auth header on a request packet following WHOAREYOU. +// makeHandshakeAuth creates the auth header on a request packet following WHOAREYOU. func (c *Codec) makeHandshakeAuth(toID enode.ID, addr string, challenge *Whoareyou) (*handshakeAuthData, *session, error) { auth := new(handshakeAuthData) auth.h.SrcID = c.localnode.ID() @@ -379,7 +379,7 @@ func (c *Codec) makeHandshakeAuth(toID enode.ID, addr string, challenge *Whoarey return auth, sec, err } -// encodeMessage encodes an encrypted message packet. +// encodeMessageHeader encodes an encrypted message packet. func (c *Codec) encodeMessageHeader(toID enode.ID, s *session) (Header, error) { head := c.makeHeader(toID, flagMessage, 0) @@ -596,7 +596,7 @@ func (c *Codec) decodeMessage(fromAddr string, head *Header, headerData, msgData // Try decrypting the message. key := c.sc.readKey(auth.SrcID, fromAddr) msg, err := c.decryptMessage(msgData, head.Nonce[:], headerData, key) - if err == errMessageDecrypt { + if errors.Is(err, errMessageDecrypt) { // It didn't work. Start the handshake since this is an ordinary message packet. return &Unknown{Nonce: head.Nonce}, nil } @@ -632,7 +632,7 @@ func (h *StaticHeader) checkValid(packetLen int) error { return nil } -// headerMask returns a cipher for 'masking' / 'unmasking' packet headers. +// mask returns a cipher for 'masking' / 'unmasking' packet headers. func (h *Header) mask(destID enode.ID) cipher.Stream { block, err := aes.NewCipher(destID[:16]) if err != nil { diff --git a/p2p/dnsdisc/client.go b/p2p/dnsdisc/client.go index 93868b39a8d4b..3f914d6e94164 100644 --- a/p2p/dnsdisc/client.go +++ b/p2p/dnsdisc/client.go @@ -19,6 +19,7 @@ package dnsdisc import ( "bytes" "context" + "errors" "fmt" "math/rand" "net" @@ -204,7 +205,7 @@ func (c *Client) doResolveEntry(ctx context.Context, domain, hash string) (entry } for _, txt := range txts { e, err := parseEntry(txt, c.cfg.ValidSchemes) - if err == errUnknownEntry { + if errors.Is(err, errUnknownEntry) { continue } if !bytes.HasPrefix(crypto.Keccak256([]byte(txt)), wantHash) { @@ -281,7 +282,7 @@ func (it *randomIterator) nextNode() *enode.Node { } n, err := ct.syncRandom(it.ctx) if err != nil { - if err == it.ctx.Err() { + if errors.Is(err, it.ctx.Err()) { return nil // context canceled. } it.c.cfg.Logger.Debug("Error in DNS random node sync", "tree", ct.loc.domain, "err", err) diff --git a/p2p/enr/entries.go b/p2p/enr/entries.go index f2118401afb85..a8b0a3839bda5 100644 --- a/p2p/enr/entries.go +++ b/p2p/enr/entries.go @@ -17,6 +17,7 @@ package enr import ( + "errors" "fmt" "io" "net" @@ -180,9 +181,16 @@ func (err *KeyError) Error() string { return fmt.Sprintf("ENR key %q: %v", err.Key, err.Err) } +func (err *KeyError) Unwrap() error { + return err.Err +} + // IsNotFound reports whether the given error means that a key/value pair is // missing from a record. func IsNotFound(err error) bool { - kerr, ok := err.(*KeyError) - return ok && kerr.Err == errNotFound + var ke *KeyError + if errors.As(err, &ke) { + return ke.Err == errNotFound + } + return false } diff --git a/p2p/msgrate/msgrate.go b/p2p/msgrate/msgrate.go index 5bfa27b43378a..d4e0eb8b5aa3a 100644 --- a/p2p/msgrate/msgrate.go +++ b/p2p/msgrate/msgrate.go @@ -111,7 +111,7 @@ const tuningImpact = 0.25 // local link is saturated. In that case, the live measurements will force us // to reduce request sizes until the throughput gets stable. // -// Lastly, message rate measurements allows us to detect if a peer is unsuaully +// Lastly, message rate measurements allows us to detect if a peer is unusually // slow compared to other peers, in which case we can decide to keep it around // or free up the slot so someone closer. // @@ -127,7 +127,7 @@ type Tracker struct { // in their sizes. // // Callers of course are free to use the item counter as a byte counter if - // or when their protocol of choise if capped by bytes instead of items. + // or when their protocol of choice if capped by bytes instead of items. // (eg. eth.getHeaders vs snap.getAccountRange). capacity map[uint64]float64 @@ -157,7 +157,7 @@ func NewTracker(caps map[uint64]float64, rtt time.Duration) *Tracker { } // Capacity calculates the number of items the peer is estimated to be able to -// retrieve within the alloted time slot. The method will round up any division +// retrieve within the allotted time slot. The method will round up any division // errors and will add an additional overestimation ratio on top. The reason for // overshooting the capacity is because certain message types might not increase // the load proportionally to the requested items, so fetching a bit more might diff --git a/p2p/nat/natupnp.go b/p2p/nat/natupnp.go index 1f5d714664504..a8de00e978b92 100644 --- a/p2p/nat/natupnp.go +++ b/p2p/nat/natupnp.go @@ -79,7 +79,7 @@ func (n *upnp) ExternalIP() (addr net.IP, err error) { func (n *upnp) AddMapping(protocol string, extport, intport int, desc string, lifetime time.Duration) error { ip, err := n.internalAddress() if err != nil { - return nil + return nil // TODO: Shouldn't we return the error? } protocol = strings.ToUpper(protocol) lifetimeS := uint32(lifetime / time.Second) diff --git a/p2p/netutil/error_test.go b/p2p/netutil/error_test.go index 645e48f837412..84d5c2c206211 100644 --- a/p2p/netutil/error_test.go +++ b/p2p/netutil/error_test.go @@ -66,7 +66,6 @@ func TestIsPacketTooBig(t *testing.T) { for i := range buf { if buf[i] != byte(i) { t.Fatalf("error in pattern") - break } } } diff --git a/p2p/peer.go b/p2p/peer.go index 257027a5b74d1..469a1b797416d 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -416,7 +416,7 @@ func (p *Peer) startProtocols(writeStart <-chan struct{}, writeErr chan<- error) if err == nil { p.log.Trace(fmt.Sprintf("Protocol %s/%d returned", proto.Name, proto.Version)) err = errProtocolReturned - } else if err != io.EOF { + } else if !errors.Is(err, io.EOF) { p.log.Trace(fmt.Sprintf("Protocol %s/%d failed", proto.Name, proto.Version), "err", err) } p.protoErr <- err diff --git a/p2p/peer_error.go b/p2p/peer_error.go index 3028685041fe3..ebc59de251a8b 100644 --- a/p2p/peer_error.go +++ b/p2p/peer_error.go @@ -103,7 +103,7 @@ func discReasonForError(err error) DiscReason { if reason, ok := err.(DiscReason); ok { return reason } - if err == errProtocolReturned { + if errors.Is(err, errProtocolReturned) { return DiscQuitting } peerError, ok := err.(*peerError) diff --git a/p2p/server.go b/p2p/server.go index 138975e54bf5f..19f7935ffcaeb 100644 --- a/p2p/server.go +++ b/p2p/server.go @@ -126,7 +126,7 @@ type Config struct { // Protocols should contain the protocols supported // by the server. Matching protocols are launched for // each peer. - Protocols []Protocol `toml:"-"` + Protocols []Protocol `toml:"-" json:"-"` // If ListenAddr is set to a non-nil address, the server // will listen for incoming connections. @@ -136,6 +136,10 @@ type Config struct { // the server is started. ListenAddr string + // If DiscAddr is set to a non-nil value, the server will use ListenAddr + // for TCP and DiscAddr for the UDP discovery protocol. + DiscAddr string + // If set to a non-nil value, the given NAT port mapper // is used to make the listening port available to the // Internet. @@ -549,7 +553,15 @@ func (srv *Server) setupDiscovery() error { return nil } - addr, err := net.ResolveUDPAddr("udp", srv.ListenAddr) + listenAddr := srv.ListenAddr + + // Use an alternate listening address for UDP if + // a custom discovery address is configured. + if srv.DiscAddr != "" { + listenAddr = srv.DiscAddr + } + + addr, err := net.ResolveUDPAddr("udp", listenAddr) if err != nil { return err } diff --git a/p2p/simulations/adapters/exec.go b/p2p/simulations/adapters/exec.go index 35ccdfb068829..7bfa8aab6d103 100644 --- a/p2p/simulations/adapters/exec.go +++ b/p2p/simulations/adapters/exec.go @@ -501,7 +501,6 @@ func startExecNodeStack() (*node.Node, error) { // Add the snapshot API. stack.RegisterAPIs([]rpc.API{{ Namespace: "simulation", - Version: "1.0", Service: SnapshotAPI{services}, }}) diff --git a/p2p/simulations/http.go b/p2p/simulations/http.go index 341ff8718b7df..b221a0597fc48 100644 --- a/p2p/simulations/http.go +++ b/p2p/simulations/http.go @@ -21,6 +21,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "html" "io" @@ -366,7 +367,6 @@ func (s *Server) StopMocker(w http.ResponseWriter, req *http.Request) { // GetMockerList returns a list of available mockers func (s *Server) GetMockers(w http.ResponseWriter, req *http.Request) { - list := GetMockerList() s.JSON(w, http.StatusOK, list) } @@ -441,6 +441,7 @@ func (s *Server) StreamNetworkEvents(w http.ResponseWriter, req *http.Request) { } } for _, conn := range snap.Conns { + conn := conn event := NewEvent(&conn) if err := writeEvent(event); err != nil { writeErr(err) @@ -559,7 +560,7 @@ func (s *Server) CreateNode(w http.ResponseWriter, req *http.Request) { config := &adapters.NodeConfig{} err := json.NewDecoder(req.Body).Decode(config) - if err != nil && err != io.EOF { + if err != nil && !errors.Is(err, io.EOF) { http.Error(w, err.Error(), http.StatusBadRequest) return } diff --git a/p2p/simulations/http_test.go b/p2p/simulations/http_test.go index f5172f3f23db6..5775977a41f01 100644 --- a/p2p/simulations/http_test.go +++ b/p2p/simulations/http_test.go @@ -489,7 +489,6 @@ func (t *expectEvents) expect(events ...*Event) { } switch expected.Type { - case EventTypeNode: if event.Node == nil { t.Fatal("expected event.Node to be set") @@ -514,7 +513,6 @@ func (t *expectEvents) expect(events ...*Event) { if event.Conn.Up != expected.Conn.Up { t.Fatalf("expected conn event %d to have up=%t, got up=%t", i, expected.Conn.Up, event.Conn.Up) } - } i++ diff --git a/p2p/simulations/mocker.go b/p2p/simulations/mocker.go index fd25e2c918ddb..5a74b02c43474 100644 --- a/p2p/simulations/mocker.go +++ b/p2p/simulations/mocker.go @@ -157,7 +157,6 @@ func probabilistic(net *Network, quit chan struct{}, nodeCount int) { } wg.Wait() } - } //connect nodeCount number of nodes in a ring diff --git a/p2p/simulations/network.go b/p2p/simulations/network.go index 962910dd25bf7..d6c5aca73c5c0 100644 --- a/p2p/simulations/network.go +++ b/p2p/simulations/network.go @@ -235,7 +235,6 @@ func (net *Network) watchPeerEvents(id enode.ID, events chan *p2p.PeerEvent, sub } peer := event.Peer switch event.Type { - case p2p.PeerEventTypeAdd: net.DidConnect(id, peer) @@ -247,7 +246,6 @@ func (net *Network) watchPeerEvents(id enode.ID, events chan *p2p.PeerEvent, sub case p2p.PeerEventTypeMsgRecv: net.DidReceive(peer, id, event.Protocol, *event.MsgCode) - } case err := <-sub.Err(): @@ -927,7 +925,6 @@ func (net *Network) snapshot(addServices []string, removeServices []string) (*Sn if !haveSvc { cleanedServices = append(cleanedServices, svc) } - } snap.Nodes[i].Node.Config.Lifecycles = cleanedServices } @@ -1021,7 +1018,6 @@ func (net *Network) Load(snap *Snapshot) error { // Start connecting. for _, conn := range snap.Conns { - if !net.GetNode(conn.One).Up() || !net.GetNode(conn.Other).Up() { //in this case, at least one of the nodes of a connection is not up, //so it would result in the snapshot `Load` to fail diff --git a/p2p/simulations/network_test.go b/p2p/simulations/network_test.go index fa6936d273c57..ab8cf19462e78 100644 --- a/p2p/simulations/network_test.go +++ b/p2p/simulations/network_test.go @@ -36,7 +36,6 @@ import ( // Tests that a created snapshot with a minimal service only contains the expected connections // and that a network when loaded with this snapshot only contains those same connections func TestSnapshot(t *testing.T) { - // PART I // create snapshot from ring network @@ -204,7 +203,6 @@ OuterTwo: t.Fatal(ctx.Err()) case ev := <-evC: if ev.Type == EventTypeConn && !ev.Control { - // fail on any disconnect if !ev.Conn.Up { t.Fatalf("unexpected disconnect: %v -> %v", ev.Conn.One, ev.Conn.Other) @@ -693,7 +691,6 @@ func BenchmarkMinimalService(b *testing.B) { } func benchmarkMinimalServiceTmp(b *testing.B) { - // stop timer to discard setup time pollution args := strings.Split(b.Name(), "/") nodeCount, err := strconv.ParseInt(args[2], 10, 16) diff --git a/p2p/tracker/tracker.go b/p2p/tracker/tracker.go index 69a49087e2c44..6a733b9ba51e3 100644 --- a/p2p/tracker/tracker.go +++ b/p2p/tracker/tracker.go @@ -121,7 +121,7 @@ func (t *Tracker) Track(peer string, version uint, reqCode uint64, resCode uint6 } // clean is called automatically when a preset time passes without a response -// being dleivered for the first network request. +// being delivered for the first network request. func (t *Tracker) clean() { t.lock.Lock() defer t.lock.Unlock() diff --git a/params/bootnodes.go b/params/bootnodes.go index 2ad230268bc68..b809977745363 100644 --- a/params/bootnodes.go +++ b/params/bootnodes.go @@ -116,6 +116,8 @@ func KnownDNSNetwork(genesis common.Hash, protocol string) string { net = "rinkeby" case GoerliGenesisHash: net = "goerli" + case SepoliaGenesisHash: + net = "sepolia" default: return "" } diff --git a/params/config.go b/params/config.go index 3533bf1c7918a..80e671f9bf417 100644 --- a/params/config.go +++ b/params/config.go @@ -55,25 +55,30 @@ var CheckpointOracles = map[common.Hash]*CheckpointOracleConfig{ } var ( + MainnetTerminalTotalDifficulty, _ = new(big.Int).SetString("58_750_000_000_000_000_000_000", 0) + // MainnetChainConfig is the chain parameters to run a node on the main network. MainnetChainConfig = &ChainConfig{ - ChainID: big.NewInt(1), - HomesteadBlock: big.NewInt(1_150_000), - DAOForkBlock: big.NewInt(1_920_000), - DAOForkSupport: true, - EIP150Block: big.NewInt(2_463_000), - EIP150Hash: common.HexToHash("0x2086799aeebeae135c246c65021c82b4e15a2c451340993aacfd2751886514f0"), - EIP155Block: big.NewInt(2_675_000), - EIP158Block: big.NewInt(2_675_000), - ByzantiumBlock: big.NewInt(4_370_000), - ConstantinopleBlock: big.NewInt(7_280_000), - PetersburgBlock: big.NewInt(7_280_000), - IstanbulBlock: big.NewInt(9_069_000), - MuirGlacierBlock: big.NewInt(9_200_000), - BerlinBlock: big.NewInt(12_244_000), - LondonBlock: big.NewInt(12_965_000), - ArrowGlacierBlock: big.NewInt(13_773_000), - Ethash: new(EthashConfig), + ChainID: big.NewInt(1), + HomesteadBlock: big.NewInt(1_150_000), + DAOForkBlock: big.NewInt(1_920_000), + DAOForkSupport: true, + EIP150Block: big.NewInt(2_463_000), + EIP150Hash: common.HexToHash("0x2086799aeebeae135c246c65021c82b4e15a2c451340993aacfd2751886514f0"), + EIP155Block: big.NewInt(2_675_000), + EIP158Block: big.NewInt(2_675_000), + ByzantiumBlock: big.NewInt(4_370_000), + ConstantinopleBlock: big.NewInt(7_280_000), + PetersburgBlock: big.NewInt(7_280_000), + IstanbulBlock: big.NewInt(9_069_000), + MuirGlacierBlock: big.NewInt(9_200_000), + BerlinBlock: big.NewInt(12_244_000), + LondonBlock: big.NewInt(12_965_000), + ArrowGlacierBlock: big.NewInt(13_773_000), + GrayGlacierBlock: big.NewInt(15_050_000), + TerminalTotalDifficulty: MainnetTerminalTotalDifficulty, // 58_750_000_000_000_000_000_000 + TerminalTotalDifficultyPassed: true, + Ethash: new(EthashConfig), } // MainnetTrustedCheckpoint contains the light client trusted checkpoint for the main network. @@ -99,23 +104,24 @@ var ( // RopstenChainConfig contains the chain parameters to run a node on the Ropsten test network. RopstenChainConfig = &ChainConfig{ - ChainID: big.NewInt(3), - HomesteadBlock: big.NewInt(0), - DAOForkBlock: nil, - DAOForkSupport: true, - EIP150Block: big.NewInt(0), - EIP150Hash: common.HexToHash("0x41941023680923e0fe4d74a34bdac8141f2540e3ae90623718e47d66d1ca4a2d"), - EIP155Block: big.NewInt(10), - EIP158Block: big.NewInt(10), - ByzantiumBlock: big.NewInt(1_700_000), - ConstantinopleBlock: big.NewInt(4_230_000), - PetersburgBlock: big.NewInt(4_939_394), - IstanbulBlock: big.NewInt(6_485_846), - MuirGlacierBlock: big.NewInt(7_117_117), - BerlinBlock: big.NewInt(9_812_189), - LondonBlock: big.NewInt(10_499_401), - TerminalTotalDifficulty: big.NewInt(43531756765713534), - Ethash: new(EthashConfig), + ChainID: big.NewInt(3), + HomesteadBlock: big.NewInt(0), + DAOForkBlock: nil, + DAOForkSupport: true, + EIP150Block: big.NewInt(0), + EIP150Hash: common.HexToHash("0x41941023680923e0fe4d74a34bdac8141f2540e3ae90623718e47d66d1ca4a2d"), + EIP155Block: big.NewInt(10), + EIP158Block: big.NewInt(10), + ByzantiumBlock: big.NewInt(1_700_000), + ConstantinopleBlock: big.NewInt(4_230_000), + PetersburgBlock: big.NewInt(4_939_394), + IstanbulBlock: big.NewInt(6_485_846), + MuirGlacierBlock: big.NewInt(7_117_117), + BerlinBlock: big.NewInt(9_812_189), + LondonBlock: big.NewInt(10_499_401), + TerminalTotalDifficulty: new(big.Int).SetUint64(50_000_000_000_000_000), + TerminalTotalDifficultyPassed: true, + Ethash: new(EthashConfig), } // RopstenTrustedCheckpoint contains the light client trusted checkpoint for the Ropsten test network. @@ -141,21 +147,24 @@ var ( // SepoliaChainConfig contains the chain parameters to run a node on the Sepolia test network. SepoliaChainConfig = &ChainConfig{ - ChainID: big.NewInt(11155111), - HomesteadBlock: big.NewInt(0), - DAOForkBlock: nil, - DAOForkSupport: true, - EIP150Block: big.NewInt(0), - EIP155Block: big.NewInt(0), - EIP158Block: big.NewInt(0), - ByzantiumBlock: big.NewInt(0), - ConstantinopleBlock: big.NewInt(0), - PetersburgBlock: big.NewInt(0), - IstanbulBlock: big.NewInt(0), - MuirGlacierBlock: big.NewInt(0), - BerlinBlock: big.NewInt(0), - LondonBlock: big.NewInt(0), - Ethash: new(EthashConfig), + ChainID: big.NewInt(11155111), + HomesteadBlock: big.NewInt(0), + DAOForkBlock: nil, + DAOForkSupport: true, + EIP150Block: big.NewInt(0), + EIP155Block: big.NewInt(0), + EIP158Block: big.NewInt(0), + ByzantiumBlock: big.NewInt(0), + ConstantinopleBlock: big.NewInt(0), + PetersburgBlock: big.NewInt(0), + IstanbulBlock: big.NewInt(0), + MuirGlacierBlock: big.NewInt(0), + BerlinBlock: big.NewInt(0), + LondonBlock: big.NewInt(0), + TerminalTotalDifficulty: big.NewInt(17_000_000_000_000_000), + TerminalTotalDifficultyPassed: true, + MergeNetsplitBlock: big.NewInt(1735371), + Ethash: new(EthashConfig), } // SepoliaTrustedCheckpoint contains the light client trusted checkpoint for the Sepolia test network. @@ -212,21 +221,23 @@ var ( // GoerliChainConfig contains the chain parameters to run a node on the Görli test network. GoerliChainConfig = &ChainConfig{ - ChainID: big.NewInt(5), - HomesteadBlock: big.NewInt(0), - DAOForkBlock: nil, - DAOForkSupport: true, - EIP150Block: big.NewInt(0), - EIP155Block: big.NewInt(0), - EIP158Block: big.NewInt(0), - ByzantiumBlock: big.NewInt(0), - ConstantinopleBlock: big.NewInt(0), - PetersburgBlock: big.NewInt(0), - IstanbulBlock: big.NewInt(1_561_651), - MuirGlacierBlock: nil, - BerlinBlock: big.NewInt(4_460_644), - LondonBlock: big.NewInt(5_062_605), - ArrowGlacierBlock: nil, + ChainID: big.NewInt(5), + HomesteadBlock: big.NewInt(0), + DAOForkBlock: nil, + DAOForkSupport: true, + EIP150Block: big.NewInt(0), + EIP155Block: big.NewInt(0), + EIP158Block: big.NewInt(0), + ByzantiumBlock: big.NewInt(0), + ConstantinopleBlock: big.NewInt(0), + PetersburgBlock: big.NewInt(0), + IstanbulBlock: big.NewInt(1_561_651), + MuirGlacierBlock: nil, + BerlinBlock: big.NewInt(4_460_644), + LondonBlock: big.NewInt(5_062_605), + ArrowGlacierBlock: nil, + TerminalTotalDifficulty: big.NewInt(10_790_000), + TerminalTotalDifficultyPassed: true, Clique: &CliqueConfig{ Period: 15, Epoch: 30000, @@ -259,19 +270,28 @@ var ( // // This configuration is intentionally not using keyed fields to force anyone // adding flags to the config to also have to set these fields. - AllEthashProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, new(EthashConfig), nil} + AllEthashProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, nil, nil, false, new(EthashConfig), nil} // AllCliqueProtocolChanges contains every protocol change (EIPs) introduced // and accepted by the Ethereum core developers into the Clique consensus. // // This configuration is intentionally not using keyed fields to force anyone // adding flags to the config to also have to set these fields. - AllCliqueProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, nil, nil, &CliqueConfig{Period: 0, Epoch: 30000}} + AllCliqueProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, nil, nil, nil, nil, false, nil, &CliqueConfig{Period: 0, Epoch: 30000}} - TestChainConfig = &ChainConfig{big.NewInt(1), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, new(EthashConfig), nil} + TestChainConfig = &ChainConfig{big.NewInt(1), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, nil, nil, false, new(EthashConfig), nil} TestRules = TestChainConfig.Rules(new(big.Int), false) ) +// NetworkNames are user friendly names to use in the chain spec banner. +var NetworkNames = map[string]string{ + MainnetChainConfig.ChainID.String(): "mainnet", + RopstenChainConfig.ChainID.String(): "ropsten", + RinkebyChainConfig.ChainID.String(): "rinkeby", + GoerliChainConfig.ChainID.String(): "goerli", + SepoliaChainConfig.ChainID.String(): "sepolia", +} + // TrustedCheckpoint represents a set of post-processed trie roots (CHT and // BloomTrie) associated with the appropriate section index and head hash. It is // used to start light syncing from this checkpoint and avoid downloading the @@ -348,12 +368,20 @@ type ChainConfig struct { BerlinBlock *big.Int `json:"berlinBlock,omitempty"` // Berlin switch block (nil = no fork, 0 = already on berlin) LondonBlock *big.Int `json:"londonBlock,omitempty"` // London switch block (nil = no fork, 0 = already on london) ArrowGlacierBlock *big.Int `json:"arrowGlacierBlock,omitempty"` // Eip-4345 (bomb delay) switch block (nil = no fork, 0 = already activated) - MergeForkBlock *big.Int `json:"mergeForkBlock,omitempty"` // EIP-3675 (TheMerge) switch block (nil = no fork, 0 = already in merge proceedings) + GrayGlacierBlock *big.Int `json:"grayGlacierBlock,omitempty"` // Eip-5133 (bomb delay) switch block (nil = no fork, 0 = already activated) + MergeNetsplitBlock *big.Int `json:"mergeNetsplitBlock,omitempty"` // Virtual fork after The Merge to use as a network splitter + ShanghaiBlock *big.Int `json:"shanghaiBlock,omitempty"` // Shanghai switch block (nil = no fork, 0 = already on shanghai) + CancunBlock *big.Int `json:"cancunBlock,omitempty"` // Cancun switch block (nil = no fork, 0 = already on cancun) // TerminalTotalDifficulty is the amount of total difficulty reached by // the network that triggers the consensus upgrade. TerminalTotalDifficulty *big.Int `json:"terminalTotalDifficulty,omitempty"` + // TerminalTotalDifficultyPassed is a flag specifying that the network already + // passed the terminal total difficulty. Its purpose is to disable legacy sync + // even without having seen the TTD locally (safer long term). + TerminalTotalDifficultyPassed bool `json:"terminalTotalDifficultyPassed,omitempty"` + // Various consensus engines Ethash *EthashConfig `json:"ethash,omitempty"` Clique *CliqueConfig `json:"clique,omitempty"` @@ -380,35 +408,82 @@ func (c *CliqueConfig) String() string { // String implements the fmt.Stringer interface. func (c *ChainConfig) String() string { - var engine interface{} + var banner string + + // Create some basinc network config output + network := NetworkNames[c.ChainID.String()] + if network == "" { + network = "unknown" + } + banner += fmt.Sprintf("Chain ID: %v (%s)\n", c.ChainID, network) switch { case c.Ethash != nil: - engine = c.Ethash + if c.TerminalTotalDifficulty == nil { + banner += "Consensus: Ethash (proof-of-work)\n" + } else if !c.TerminalTotalDifficultyPassed { + banner += "Consensus: Beacon (proof-of-stake), merging from Ethash (proof-of-work)\n" + } else { + banner += "Consensus: Beacon (proof-of-stake), merged from Ethash (proof-of-work)\n" + } case c.Clique != nil: - engine = c.Clique + if c.TerminalTotalDifficulty == nil { + banner += "Consensus: Clique (proof-of-authority)\n" + } else if !c.TerminalTotalDifficultyPassed { + banner += "Consensus: Beacon (proof-of-stake), merging from Clique (proof-of-authority)\n" + } else { + banner += "Consensus: Beacon (proof-of-stake), merged from Clique (proof-of-authority)\n" + } default: - engine = "unknown" - } - return fmt.Sprintf("{ChainID: %v Homestead: %v DAO: %v DAOSupport: %v EIP150: %v EIP155: %v EIP158: %v Byzantium: %v Constantinople: %v Petersburg: %v Istanbul: %v, Muir Glacier: %v, Berlin: %v, London: %v, Arrow Glacier: %v, MergeFork: %v, Terminal TD: %v, Engine: %v}", - c.ChainID, - c.HomesteadBlock, - c.DAOForkBlock, - c.DAOForkSupport, - c.EIP150Block, - c.EIP155Block, - c.EIP158Block, - c.ByzantiumBlock, - c.ConstantinopleBlock, - c.PetersburgBlock, - c.IstanbulBlock, - c.MuirGlacierBlock, - c.BerlinBlock, - c.LondonBlock, - c.ArrowGlacierBlock, - c.MergeForkBlock, - c.TerminalTotalDifficulty, - engine, - ) + banner += "Consensus: unknown\n" + } + banner += "\n" + + // Create a list of forks with a short description of them. Forks that only + // makes sense for mainnet should be optional at printing to avoid bloating + // the output for testnets and private networks. + banner += "Pre-Merge hard forks:\n" + banner += fmt.Sprintf(" - Homestead: %-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/homestead.md)\n", c.HomesteadBlock) + if c.DAOForkBlock != nil { + banner += fmt.Sprintf(" - DAO Fork: %-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/dao-fork.md)\n", c.DAOForkBlock) + } + banner += fmt.Sprintf(" - Tangerine Whistle (EIP 150): %-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/tangerine-whistle.md)\n", c.EIP150Block) + banner += fmt.Sprintf(" - Spurious Dragon/1 (EIP 155): %-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/spurious-dragon.md)\n", c.EIP155Block) + banner += fmt.Sprintf(" - Spurious Dragon/2 (EIP 158): %-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/spurious-dragon.md)\n", c.EIP155Block) + banner += fmt.Sprintf(" - Byzantium: %-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/byzantium.md)\n", c.ByzantiumBlock) + banner += fmt.Sprintf(" - Constantinople: %-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/constantinople.md)\n", c.ConstantinopleBlock) + banner += fmt.Sprintf(" - Petersburg: %-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/petersburg.md)\n", c.PetersburgBlock) + banner += fmt.Sprintf(" - Istanbul: %-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/istanbul.md)\n", c.IstanbulBlock) + if c.MuirGlacierBlock != nil { + banner += fmt.Sprintf(" - Muir Glacier: %-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/muir-glacier.md)\n", c.MuirGlacierBlock) + } + banner += fmt.Sprintf(" - Berlin: %-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/berlin.md)\n", c.BerlinBlock) + banner += fmt.Sprintf(" - London: %-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/london.md)\n", c.LondonBlock) + if c.ArrowGlacierBlock != nil { + banner += fmt.Sprintf(" - Arrow Glacier: %-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/arrow-glacier.md)\n", c.ArrowGlacierBlock) + } + if c.GrayGlacierBlock != nil { + banner += fmt.Sprintf(" - Gray Glacier: %-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/gray-glacier.md)\n", c.GrayGlacierBlock) + } + if c.ShanghaiBlock != nil { + banner += fmt.Sprintf(" - Shanghai: %-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/shanghai.md)\n", c.ShanghaiBlock) + } + if c.CancunBlock != nil { + banner += fmt.Sprintf(" - Cancun: %-8v\n", c.CancunBlock) + } + banner += "\n" + + // Add a special section for the merge as it's non-obvious + if c.TerminalTotalDifficulty == nil { + banner += "The Merge is not yet available for this network!\n" + banner += " - Hard-fork specification: https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/paris.md" + } else { + banner += "Merge configured:\n" + banner += " - Hard-fork specification: https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/paris.md\n" + banner += fmt.Sprintf(" - Network known to be merged: %v\n", c.TerminalTotalDifficultyPassed) + banner += fmt.Sprintf(" - Total terminal difficulty: %v\n", c.TerminalTotalDifficulty) + banner += fmt.Sprintf(" - Merge netsplit block: %-8v", c.MergeNetsplitBlock) + } + return banner } // IsHomestead returns whether num is either equal to the homestead block or greater. @@ -478,6 +553,11 @@ func (c *ChainConfig) IsArrowGlacier(num *big.Int) bool { return isForked(c.ArrowGlacierBlock, num) } +// IsGrayGlacier returns whether num is either equal to the Gray Glacier (EIP-5133) fork block or greater. +func (c *ChainConfig) IsGrayGlacier(num *big.Int) bool { + return isForked(c.GrayGlacierBlock, num) +} + // IsTerminalPoWBlock returns whether the given block is the last block of PoW stage. func (c *ChainConfig) IsTerminalPoWBlock(parentTotalDiff *big.Int, totalDiff *big.Int) bool { if c.TerminalTotalDifficulty == nil { @@ -486,6 +566,16 @@ func (c *ChainConfig) IsTerminalPoWBlock(parentTotalDiff *big.Int, totalDiff *bi return parentTotalDiff.Cmp(c.TerminalTotalDifficulty) < 0 && totalDiff.Cmp(c.TerminalTotalDifficulty) >= 0 } +// IsShanghai returns whether num is either equal to the Shanghai fork block or greater. +func (c *ChainConfig) IsShanghai(num *big.Int) bool { + return isForked(c.ShanghaiBlock, num) +} + +// IsCancun returns whether num is either equal to the Cancun fork block or greater. +func (c *ChainConfig) IsCancun(num *big.Int) bool { + return isForked(c.CancunBlock, num) +} + // CheckCompatible checks whether scheduled fork transitions have been imported // with a mismatching chain configuration. func (c *ChainConfig) CheckCompatible(newcfg *ChainConfig, height uint64) *ConfigCompatError { @@ -527,7 +617,10 @@ func (c *ChainConfig) CheckConfigForkOrder() error { {name: "berlinBlock", block: c.BerlinBlock}, {name: "londonBlock", block: c.LondonBlock}, {name: "arrowGlacierBlock", block: c.ArrowGlacierBlock, optional: true}, - {name: "mergeStartBlock", block: c.MergeForkBlock, optional: true}, + {name: "grayGlacierBlock", block: c.GrayGlacierBlock, optional: true}, + {name: "mergeNetsplitBlock", block: c.MergeNetsplitBlock, optional: true}, + {name: "shanghaiBlock", block: c.ShanghaiBlock, optional: true}, + {name: "cancunBlock", block: c.CancunBlock, optional: true}, } { if lastFork.name != "" { // Next one must be higher number @@ -600,8 +693,17 @@ func (c *ChainConfig) checkCompatible(newcfg *ChainConfig, head *big.Int) *Confi if isForkIncompatible(c.ArrowGlacierBlock, newcfg.ArrowGlacierBlock, head) { return newCompatError("Arrow Glacier fork block", c.ArrowGlacierBlock, newcfg.ArrowGlacierBlock) } - if isForkIncompatible(c.MergeForkBlock, newcfg.MergeForkBlock, head) { - return newCompatError("Merge Start fork block", c.MergeForkBlock, newcfg.MergeForkBlock) + if isForkIncompatible(c.GrayGlacierBlock, newcfg.GrayGlacierBlock, head) { + return newCompatError("Gray Glacier fork block", c.GrayGlacierBlock, newcfg.GrayGlacierBlock) + } + if isForkIncompatible(c.MergeNetsplitBlock, newcfg.MergeNetsplitBlock, head) { + return newCompatError("Merge netsplit fork block", c.MergeNetsplitBlock, newcfg.MergeNetsplitBlock) + } + if isForkIncompatible(c.ShanghaiBlock, newcfg.ShanghaiBlock, head) { + return newCompatError("Shanghai fork block", c.ShanghaiBlock, newcfg.ShanghaiBlock) + } + if isForkIncompatible(c.CancunBlock, newcfg.CancunBlock, head) { + return newCompatError("Cancun fork block", c.CancunBlock, newcfg.CancunBlock) } return nil } @@ -671,7 +773,7 @@ type Rules struct { IsHomestead, IsEIP150, IsEIP155, IsEIP158 bool IsByzantium, IsConstantinople, IsPetersburg, IsIstanbul bool IsBerlin, IsLondon bool - IsMerge bool + IsMerge, IsShanghai, isCancun bool } // Rules ensures c's ChainID is not nil. @@ -693,5 +795,7 @@ func (c *ChainConfig) Rules(num *big.Int, isMerge bool) Rules { IsBerlin: c.IsBerlin(num), IsLondon: c.IsLondon(num), IsMerge: isMerge, + IsShanghai: c.IsShanghai(num), + isCancun: c.IsCancun(num), } } diff --git a/params/version.go b/params/version.go index 41b4e46c3d68c..d9a3958512c34 100644 --- a/params/version.go +++ b/params/version.go @@ -23,7 +23,7 @@ import ( const ( VersionMajor = 1 // Major version component of the current release VersionMinor = 10 // Minor version component of the current release - VersionPatch = 18 // Patch version component of the current release + VersionPatch = 26 // Patch version component of the current release VersionMeta = "stable" // Version metadata to append to the version string ) @@ -41,9 +41,9 @@ var VersionWithMeta = func() string { return v }() -// ArchiveVersion holds the textual version string used for Geth archives. -// e.g. "1.8.11-dea1ce05" for stable releases, or -// "1.8.13-unstable-21c059b6" for unstable releases +// ArchiveVersion holds the textual version string used for Geth archives. e.g. +// "1.8.11-dea1ce05" for stable releases, or "1.8.13-unstable-21c059b6" for unstable +// releases. func ArchiveVersion(gitCommit string) string { vsn := Version if VersionMeta != "stable" { diff --git a/rlp/decode_test.go b/rlp/decode_test.go index 46aa68cea3d73..00722f847bbbc 100644 --- a/rlp/decode_test.go +++ b/rlp/decode_test.go @@ -452,7 +452,7 @@ type ignoredField struct { var ( veryBigInt = new(big.Int).Add( - big.NewInt(0).Lsh(big.NewInt(0xFFFFFFFFFFFFFF), 16), + new(big.Int).Lsh(big.NewInt(0xFFFFFFFFFFFFFF), 16), big.NewInt(0xFFFF), ) veryVeryBigInt = new(big.Int).Exp(veryBigInt, big.NewInt(8), nil) @@ -1043,7 +1043,6 @@ func TestInvalidOptionalField(t *testing.T) { t.Errorf("wrong error for %T: %v", test.v, err.Error()) } } - } func ExampleDecode() { diff --git a/rlp/encode_test.go b/rlp/encode_test.go index 78392906b557e..58ddc0d120f0f 100644 --- a/rlp/encode_test.go +++ b/rlp/encode_test.go @@ -119,15 +119,15 @@ var encTests = []encTest{ {val: big.NewInt(0xFFFFFFFFFFFF), output: "86FFFFFFFFFFFF"}, {val: big.NewInt(0xFFFFFFFFFFFFFF), output: "87FFFFFFFFFFFFFF"}, { - val: big.NewInt(0).SetBytes(unhex("102030405060708090A0B0C0D0E0F2")), + val: new(big.Int).SetBytes(unhex("102030405060708090A0B0C0D0E0F2")), output: "8F102030405060708090A0B0C0D0E0F2", }, { - val: big.NewInt(0).SetBytes(unhex("0100020003000400050006000700080009000A000B000C000D000E01")), + val: new(big.Int).SetBytes(unhex("0100020003000400050006000700080009000A000B000C000D000E01")), output: "9C0100020003000400050006000700080009000A000B000C000D000E01", }, { - val: big.NewInt(0).SetBytes(unhex("010000000000000000000000000000000000000000000000000000000000000000")), + val: new(big.Int).SetBytes(unhex("010000000000000000000000000000000000000000000000000000000000000000")), output: "A1010000000000000000000000000000000000000000000000000000000000000000", }, { diff --git a/rlp/iterator.go b/rlp/iterator.go index 353ef09fbdf25..6be574572e611 100644 --- a/rlp/iterator.go +++ b/rlp/iterator.go @@ -36,7 +36,6 @@ func NewListIterator(data RawValue) (*listIterator, error) { data: data[t : t+c], } return it, nil - } // Next forwards the iterator one step, returns true if it was not at end yet diff --git a/rlp/rlpgen/main.go b/rlp/rlpgen/main.go index 6258fdb47a563..25d4393cc6561 100644 --- a/rlp/rlpgen/main.go +++ b/rlp/rlpgen/main.go @@ -51,7 +51,7 @@ func main() { } if *output == "-" { os.Stdout.Write(code) - } else if err := os.WriteFile(*output, code, 0644); err != nil { + } else if err := os.WriteFile(*output, code, 0600); err != nil { fatal(err) } } @@ -106,7 +106,7 @@ func (cfg *Config) process() (code []byte, err error) { // Find the type and generate. typ, err := lookupStructType(pkg.Scope(), cfg.Type) if err != nil { - return nil, fmt.Errorf("can't find %s in %s: %v", typ, pkg, err) + return nil, fmt.Errorf("can't find %s in %s: %v", cfg.Type, pkg, err) } code, err = bctx.generate(typ, cfg.GenerateEncoder, cfg.GenerateDecoder) if err != nil { diff --git a/rpc/client.go b/rpc/client.go index d3ce0297754c9..fcd31319a3c59 100644 --- a/rpc/client.go +++ b/rpc/client.go @@ -31,6 +31,7 @@ import ( ) var ( + ErrBadResult = errors.New("bad result in JSON-RPC response") ErrClientQuit = errors.New("client is closed") ErrNoResult = errors.New("no result in JSON-RPC response") ErrSubscriptionQueueOverflow = errors.New("subscription queue overflow") diff --git a/rpc/client_test.go b/rpc/client_test.go index fa6010bb199c1..5ae549d865336 100644 --- a/rpc/client_test.go +++ b/rpc/client_test.go @@ -19,6 +19,7 @@ package rpc import ( "context" "encoding/json" + "errors" "fmt" "math/rand" "net" @@ -144,6 +145,53 @@ func TestClientBatchRequest(t *testing.T) { } } +func TestClientBatchRequest_len(t *testing.T) { + b, err := json.Marshal([]jsonrpcMessage{ + {Version: "2.0", ID: json.RawMessage("1"), Method: "foo", Result: json.RawMessage(`"0x1"`)}, + {Version: "2.0", ID: json.RawMessage("2"), Method: "bar", Result: json.RawMessage(`"0x2"`)}, + }) + if err != nil { + t.Fatal("failed to encode jsonrpc message:", err) + } + s := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + _, err := rw.Write(b) + if err != nil { + t.Error("failed to write response:", err) + } + })) + t.Cleanup(s.Close) + + client, err := Dial(s.URL) + if err != nil { + t.Fatal("failed to dial test server:", err) + } + defer client.Close() + + t.Run("too-few", func(t *testing.T) { + batch := []BatchElem{ + {Method: "foo"}, + {Method: "bar"}, + {Method: "baz"}, + } + ctx, cancelFn := context.WithTimeout(context.Background(), time.Second) + defer cancelFn() + if err := client.BatchCallContext(ctx, batch); !errors.Is(err, ErrBadResult) { + t.Errorf("expected %q but got: %v", ErrBadResult, err) + } + }) + + t.Run("too-many", func(t *testing.T) { + batch := []BatchElem{ + {Method: "foo"}, + } + ctx, cancelFn := context.WithTimeout(context.Background(), time.Second) + defer cancelFn() + if err := client.BatchCallContext(ctx, batch); !errors.Is(err, ErrBadResult) { + t.Errorf("expected %q but got: %v", ErrBadResult, err) + } + }) +} + func TestClientNotify(t *testing.T) { server := newTestServer() defer server.Stop() @@ -615,10 +663,10 @@ func TestClientReconnect(t *testing.T) { // Start a server and corresponding client. s1, l1 := startServer("127.0.0.1:0") client, err := DialContext(ctx, "ws://"+l1.Addr().String()) - defer client.Close() if err != nil { t.Fatal("can't dial", err) } + defer client.Close() // Perform a call. This should work because the server is up. var resp echoResult diff --git a/rpc/handler.go b/rpc/handler.go index e8d1887c7d236..cd95a067f3e26 100644 --- a/rpc/handler.go +++ b/rpc/handler.go @@ -346,7 +346,7 @@ func (h *handler) handleCall(cp *callProc, msg *jsonrpcMessage) *jsonrpcMessage successfulRequestGauge.Inc(1) } rpcServingTimer.UpdateSince(start) - newRPCServingTimer(msg.Method, answer.Error == nil).UpdateSince(start) + updateServeTimeHistogram(msg.Method, answer.Error == nil, time.Since(start)) } return answer } diff --git a/rpc/http.go b/rpc/http.go index 9f44649573493..b82ea2707a546 100644 --- a/rpc/http.go +++ b/rpc/http.go @@ -87,6 +87,14 @@ type HTTPTimeouts struct { // ReadHeaderTimeout. It is valid to use them both. ReadTimeout time.Duration + // ReadHeaderTimeout is the amount of time allowed to read + // request headers. The connection's read deadline is reset + // after reading the headers and the Handler can decide what + // is considered too slow for the body. If ReadHeaderTimeout + // is zero, the value of ReadTimeout is used. If both are + // zero, there is no timeout. + ReadHeaderTimeout time.Duration + // WriteTimeout is the maximum duration before timing out // writes of the response. It is reset whenever a new // request's header is read. Like ReadTimeout, it does not @@ -103,9 +111,10 @@ type HTTPTimeouts struct { // DefaultHTTPTimeouts represents the default timeout values used if further // configuration is not provided. var DefaultHTTPTimeouts = HTTPTimeouts{ - ReadTimeout: 30 * time.Second, - WriteTimeout: 30 * time.Second, - IdleTimeout: 120 * time.Second, + ReadTimeout: 30 * time.Second, + ReadHeaderTimeout: 30 * time.Second, + WriteTimeout: 30 * time.Second, + IdleTimeout: 120 * time.Second, } // DialHTTPWithClient creates a new RPC client that connects to an RPC server over HTTP @@ -164,6 +173,9 @@ func (c *Client) sendBatchHTTP(ctx context.Context, op *requestOp, msgs []*jsonr if err := json.NewDecoder(respBody).Decode(&respmsgs); err != nil { return err } + if len(respmsgs) != len(msgs) { + return fmt.Errorf("batch has %d requests but response has %d: %w", len(msgs), len(respmsgs), ErrBadResult) + } for i := 0; i < len(respmsgs); i++ { op.resp <- &respmsgs[i] } diff --git a/rpc/metrics.go b/rpc/metrics.go index 4f166ad1cc073..b1f1284535e26 100644 --- a/rpc/metrics.go +++ b/rpc/metrics.go @@ -18,6 +18,7 @@ package rpc import ( "fmt" + "time" "github.com/ethereum/go-ethereum/metrics" ) @@ -26,14 +27,24 @@ var ( rpcRequestGauge = metrics.NewRegisteredGauge("rpc/requests", nil) successfulRequestGauge = metrics.NewRegisteredGauge("rpc/success", nil) failedRequestGauge = metrics.NewRegisteredGauge("rpc/failure", nil) - rpcServingTimer = metrics.NewRegisteredTimer("rpc/duration/all", nil) + + // serveTimeHistName is the prefix of the per-request serving time histograms. + serveTimeHistName = "rpc/duration" + + rpcServingTimer = metrics.NewRegisteredTimer("rpc/duration/all", nil) ) -func newRPCServingTimer(method string, valid bool) metrics.Timer { - flag := "success" - if !valid { - flag = "failure" +// updateServeTimeHistogram tracks the serving time of a remote RPC call. +func updateServeTimeHistogram(method string, success bool, elapsed time.Duration) { + note := "success" + if !success { + note = "failure" + } + h := fmt.Sprintf("%s/%s/%s", serveTimeHistName, method, note) + sampler := func() metrics.Sample { + return metrics.ResettingSample( + metrics.NewExpDecaySample(1028, 0.015), + ) } - m := fmt.Sprintf("rpc/duration/%s/%s", method, flag) - return metrics.GetOrRegisterTimer(m, nil) + metrics.GetOrRegisterHistogramLazy(h, nil, sampler).Update(elapsed.Microseconds()) } diff --git a/rpc/server.go b/rpc/server.go index babc5688e2648..bf1f71a28e262 100644 --- a/rpc/server.go +++ b/rpc/server.go @@ -160,7 +160,7 @@ type PeerInfo struct { // Address of client. This will usually contain the IP address and port. RemoteAddr string - // Addditional information for HTTP and WebSocket connections. + // Additional information for HTTP and WebSocket connections. HTTP struct { // Protocol version, i.e. "HTTP/1.1". This is not set for WebSocket. Version string diff --git a/rpc/subscription_test.go b/rpc/subscription_test.go index 54a053dba8052..a920205c001f9 100644 --- a/rpc/subscription_test.go +++ b/rpc/subscription_test.go @@ -48,7 +48,7 @@ func TestNewID(t *testing.T) { func TestSubscriptions(t *testing.T) { var ( - namespaces = []string{"eth", "shh", "bzz"} + namespaces = []string{"eth", "bzz"} service = ¬ificationTestService{} subCount = len(namespaces) notificationCount = 3 diff --git a/rpc/types.go b/rpc/types.go index f4d05be48cd47..e3d1a48968211 100644 --- a/rpc/types.go +++ b/rpc/types.go @@ -31,9 +31,9 @@ import ( // API describes the set of methods offered over the RPC interface type API struct { Namespace string // namespace under which the rpc methods of Service are exposed - Version string // api version for DApp's + Version string // deprecated - this field is no longer used, but retained for compatibility Service interface{} // receiver instance which holds the methods - Public bool // indication if the methods must be considered safe for public use + Public bool // deprecated - this field is no longer used, but retained for compatibility Authenticated bool // whether the api should only be available behind authentication. } @@ -61,6 +61,7 @@ type jsonWriter interface { type BlockNumber int64 const ( + SafeBlockNumber = BlockNumber(-4) FinalizedBlockNumber = BlockNumber(-3) PendingBlockNumber = BlockNumber(-2) LatestBlockNumber = BlockNumber(-1) @@ -92,6 +93,9 @@ func (bn *BlockNumber) UnmarshalJSON(data []byte) error { case "finalized": *bn = FinalizedBlockNumber return nil + case "safe": + *bn = SafeBlockNumber + return nil } blckNum, err := hexutil.DecodeUint64(input) @@ -118,6 +122,8 @@ func (bn BlockNumber) MarshalText() ([]byte, error) { return []byte("pending"), nil case FinalizedBlockNumber: return []byte("finalized"), nil + case SafeBlockNumber: + return []byte("safe"), nil default: return hexutil.Uint64(bn).MarshalText() } @@ -168,6 +174,10 @@ func (bnh *BlockNumberOrHash) UnmarshalJSON(data []byte) error { bn := FinalizedBlockNumber bnh.BlockNumber = &bn return nil + case "safe": + bn := SafeBlockNumber + bnh.BlockNumber = &bn + return nil default: if len(input) == 66 { hash := common.Hash{} diff --git a/signer/core/api.go b/signer/core/api.go index f06fbeb76dd13..f10f03d83ac1b 100644 --- a/signer/core/api.go +++ b/signer/core/api.go @@ -319,7 +319,6 @@ func (api *SignerAPI) openTrezor(url accounts.URL) { log.Warn("failed to open wallet", "wallet", url, "err", err) return } - } // startUSBListener starts a listener for USB events, for hardware wallet interaction @@ -612,7 +611,6 @@ func (api *SignerAPI) SignTransaction(ctx context.Context, args apitypes.SendTxA api.UI.OnApprovedTx(response) // ...and to the external caller return &response, nil - } func (api *SignerAPI) SignGnosisSafeTx(ctx context.Context, signerAddress common.MixedcaseAddress, gnosisTx GnosisSafeTx, methodSelector *string) (*GnosisSafeTx, error) { diff --git a/signer/core/api_test.go b/signer/core/api_test.go index ddc2b82eac69b..6fa2af1836b2e 100644 --- a/signer/core/api_test.go +++ b/signer/core/api_test.go @@ -55,14 +55,13 @@ func (ui *headlessUi) RegisterUIServer(api *core.UIServerAPI) {} func (ui *headlessUi) OnApprovedTx(tx ethapi.SignTransactionResult) {} func (ui *headlessUi) ApproveTx(request *core.SignTxRequest) (core.SignTxResponse, error) { - switch <-ui.approveCh { case "Y": return core.SignTxResponse{request.Transaction, true}, nil case "M": // modify // The headless UI always modifies the transaction old := big.Int(request.Transaction.Value) - newVal := big.NewInt(0).Add(&old, big.NewInt(1)) + newVal := new(big.Int).Add(&old, big.NewInt(1)) request.Transaction.Value = hexutil.Big(*newVal) return core.SignTxResponse{request.Transaction, true}, nil default: @@ -125,7 +124,6 @@ func setup(t *testing.T) (*core.SignerAPI, *headlessUi) { am := core.StartClefAccountManager(tmpDirName(t), true, true, "") api := core.NewSignerAPI(am, 1337, true, ui, db, true, &storage.NoStorage{}) return api, ui - } func createAccount(ui *headlessUi, api *core.SignerAPI, t *testing.T) { ui.approveCh <- "Y" @@ -139,7 +137,6 @@ func createAccount(ui *headlessUi, api *core.SignerAPI, t *testing.T) { } func failCreateAccountWithPassword(ui *headlessUi, api *core.SignerAPI, password string, t *testing.T) { - ui.approveCh <- "Y" // We will be asked three times to provide a suitable password ui.inputCh <- password @@ -169,7 +166,6 @@ func failCreateAccount(ui *headlessUi, api *core.SignerAPI, t *testing.T) { func list(ui *headlessUi, api *core.SignerAPI, t *testing.T) ([]common.Address, error) { ui.approveCh <- "A" return api.List(context.Background()) - } func TestNewAcc(t *testing.T) { @@ -321,5 +317,4 @@ func TestSignTx(t *testing.T) { if bytes.Equal(res.Raw, res2.Raw) { t.Error("Expected tx to be modified by UI") } - } diff --git a/signer/core/apitypes/types.go b/signer/core/apitypes/types.go index f5c2fe2f3db91..0652108f889eb 100644 --- a/signer/core/apitypes/types.go +++ b/signer/core/apitypes/types.go @@ -251,6 +251,25 @@ type TypedDataDomain struct { Salt string `json:"salt"` } +// TypedDataAndHash is a helper function that calculates a hash for typed data conforming to EIP-712. +// This hash can then be safely used to calculate a signature. +// +// See https://eips.ethereum.org/EIPS/eip-712 for the full specification. +// +// This gives context to the signed typed data and prevents signing of transactions. +func TypedDataAndHash(typedData TypedData) ([]byte, string, error) { + domainSeparator, err := typedData.HashStruct("EIP712Domain", typedData.Domain.Map()) + if err != nil { + return nil, "", err + } + typedDataHash, err := typedData.HashStruct(typedData.PrimaryType, typedData.Message) + if err != nil { + return nil, "", err + } + rawData := fmt.Sprintf("\x19\x01%s%s", string(domainSeparator), string(typedDataHash)) + return crypto.Keccak256([]byte(rawData)), rawData, nil +} + // HashStruct generates a keccak256 hash of the encoding of the provided data func (typedData *TypedData) HashStruct(primaryType string, data TypedDataMessage) (hexutil.Bytes, error) { encodedData, err := typedData.EncodeData(primaryType, data, 1) @@ -526,7 +545,6 @@ func (typedData *TypedData) EncodePrimitiveValue(encType string, encValue interf return math.U256Bytes(b), nil } return nil, fmt.Errorf("unrecognized type '%s'", encType) - } // dataMismatchError generates an error for a mismatch between @@ -653,13 +671,12 @@ func formatPrimitiveValue(encType string, encValue interface{}) (string, error) } if strings.HasPrefix(encType, "bytes") { return fmt.Sprintf("%s", encValue), nil - } if strings.HasPrefix(encType, "uint") || strings.HasPrefix(encType, "int") { if b, err := parseInteger(encType, encValue); err != nil { return "", err } else { - return fmt.Sprintf("%d (0x%x)", b, b), nil + return fmt.Sprintf("%d (%#x)", b, b), nil } } return "", fmt.Errorf("unhandled type %v", encType) @@ -784,6 +801,8 @@ func isPrimitiveTypeValid(primitiveType string) bool { primitiveType == "int32[]" || primitiveType == "int64" || primitiveType == "int64[]" || + primitiveType == "int96" || + primitiveType == "int96[]" || primitiveType == "int128" || primitiveType == "int128[]" || primitiveType == "int256" || @@ -800,6 +819,8 @@ func isPrimitiveTypeValid(primitiveType string) bool { primitiveType == "uint32[]" || primitiveType == "uint64" || primitiveType == "uint64[]" || + primitiveType == "uint96" || + primitiveType == "uint96[]" || primitiveType == "uint128" || primitiveType == "uint128[]" || primitiveType == "uint256" || diff --git a/signer/core/auditlog.go b/signer/core/auditlog.go index 663d6d1317354..a0b292bf714ca 100644 --- a/signer/core/auditlog.go +++ b/signer/core/auditlog.go @@ -110,7 +110,6 @@ func (l *AuditLogger) Version(ctx context.Context) (string, error) { data, err := l.api.Version(ctx) l.log.Info("Version", "type", "response", "data", data, "error", err) return data, err - } func NewAuditLogger(path string, api ExternalAPI) (*AuditLogger, error) { diff --git a/signer/core/cliui.go b/signer/core/cliui.go index 05c60906cc0cd..187eb1390af7a 100644 --- a/signer/core/cliui.go +++ b/signer/core/cliui.go @@ -59,7 +59,6 @@ func (ui *CommandlineUI) readString() string { } func (ui *CommandlineUI) OnInputRequired(info UserInputRequest) (UserInputResponse, error) { - fmt.Printf("## %s\n\n%s\n", info.Title, info.Prompt) defer fmt.Println("-----------------------") if info.IsPassword { @@ -147,7 +146,6 @@ func (ui *CommandlineUI) ApproveTx(request *SignTxRequest) (SignTxResponse, erro fmt.Printf(" * %s : %s\n", m.Typ, m.Message) } fmt.Println() - } fmt.Printf("\n") showMetadata(request.Meta) @@ -209,7 +207,6 @@ func (ui *CommandlineUI) ApproveListing(request *ListRequest) (ListResponse, err // ApproveNewAccount prompt the user for confirmation to create new Account, and reveal to caller func (ui *CommandlineUI) ApproveNewAccount(request *NewAccountRequest) (NewAccountResponse, error) { - ui.mu.Lock() defer ui.mu.Unlock() @@ -245,7 +242,6 @@ func (ui *CommandlineUI) OnApprovedTx(tx ethapi.SignTransactionResult) { } func (ui *CommandlineUI) OnSignerStartup(info StartupInfo) { - fmt.Printf("------- Signer info -------\n") for k, v := range info.Info { fmt.Printf("* %v : %v\n", k, v) diff --git a/signer/core/signed_data.go b/signer/core/signed_data.go index 9bf47be799d8d..c0da22e626623 100644 --- a/signer/core/signed_data.go +++ b/signer/core/signed_data.go @@ -129,7 +129,7 @@ func (api *SignerAPI) determineSignatureFormat(ctx context.Context, contentType { Name: "Full message for signing", Typ: "hexdata", - Value: fmt.Sprintf("0x%x", msg), + Value: fmt.Sprintf("%#x", msg), }, } req = &SignDataRequest{ContentType: mediaType, Rawdata: []byte(msg), Messages: messages, Hash: sighash} @@ -161,7 +161,7 @@ func (api *SignerAPI) determineSignatureFormat(ctx context.Context, contentType { Name: "Clique header", Typ: "clique", - Value: fmt.Sprintf("clique header %d [0x%x]", header.Number, header.Hash()), + Value: fmt.Sprintf("clique header %d [%#x]", header.Number, header.Hash()), }, } // Clique uses V on the form 0 or 1 @@ -169,7 +169,7 @@ func (api *SignerAPI) determineSignatureFormat(ctx context.Context, contentType req = &SignDataRequest{ContentType: mediaType, Rawdata: cliqueRlp, Messages: messages, Hash: sighash} default: // also case TextPlain.Mime: // Calculates an Ethereum ECDSA signature for: - // hash = keccak256("\x19${byteVersion}Ethereum Signed Message:\n${message length}${message}") + // hash = keccak256("\x19Ethereum Signed Message:\n${message length}${message}") // We expect it to be a string if stringData, ok := data.(string); !ok { return nil, useEthereumV, fmt.Errorf("input for text/plain must be an hex-encoded string") @@ -194,7 +194,7 @@ func (api *SignerAPI) determineSignatureFormat(ctx context.Context, contentType return req, useEthereumV, nil } -// SignTextWithValidator signs the given message which can be further recovered +// SignTextValidator signs the given message which can be further recovered // with the given validator. // hash = keccak256("\x19\x00"${address}${data}). func SignTextValidator(validatorData apitypes.ValidatorData) (hexutil.Bytes, string) { @@ -233,23 +233,17 @@ func (api *SignerAPI) SignTypedData(ctx context.Context, addr common.MixedcaseAd // - the signature preimage (hash) func (api *SignerAPI) signTypedData(ctx context.Context, addr common.MixedcaseAddress, typedData apitypes.TypedData, validationMessages *apitypes.ValidationMessages) (hexutil.Bytes, hexutil.Bytes, error) { - domainSeparator, err := typedData.HashStruct("EIP712Domain", typedData.Domain.Map()) + sighash, rawData, err := apitypes.TypedDataAndHash(typedData) if err != nil { return nil, nil, err } - typedDataHash, err := typedData.HashStruct(typedData.PrimaryType, typedData.Message) - if err != nil { - return nil, nil, err - } - rawData := []byte(fmt.Sprintf("\x19\x01%s%s", string(domainSeparator), string(typedDataHash))) - sighash := crypto.Keccak256(rawData) messages, err := typedData.Format() if err != nil { return nil, nil, err } req := &SignDataRequest{ ContentType: apitypes.DataTyped.Mime, - Rawdata: rawData, + Rawdata: []byte(rawData), Messages: messages, Hash: sighash, Address: addr} @@ -271,11 +265,11 @@ func (api *SignerAPI) EcRecover(ctx context.Context, data hexutil.Bytes, sig hex // // Note, this function is compatible with eth_sign and personal_sign. As such it recovers // the address of: - // hash = keccak256("\x19${byteVersion}Ethereum Signed Message:\n${message length}${message}") + // hash = keccak256("\x19Ethereum Signed Message:\n${message length}${message}") // addr = ecrecover(hash, signature) // // Note, the signature must conform to the secp256k1 curve R, S and V values, where - // the V value must be be 27 or 28 for legacy reasons. + // the V value must be 27 or 28 for legacy reasons. // // https://github.com/ethereum/go-ethereum/wiki/Management-APIs#personal_ecRecover if len(sig) != 65 { diff --git a/signer/core/validation_test.go b/signer/core/validation_test.go index 7105691d29c0c..6adaa21afd4ee 100644 --- a/signer/core/validation_test.go +++ b/signer/core/validation_test.go @@ -38,7 +38,6 @@ func TestPasswordValidation(t *testing.T) { if err == nil && test.shouldFail { t.Errorf("password '%v' should fail validation", test.pw) } else if err != nil && !test.shouldFail { - t.Errorf("password '%v' shound not fail validation, but did: %v", test.pw, err) } } diff --git a/signer/fourbyte/validation_test.go b/signer/fourbyte/validation_test.go index c3085696f4326..1b0ab507a8649 100644 --- a/signer/fourbyte/validation_test.go +++ b/signer/fourbyte/validation_test.go @@ -29,11 +29,11 @@ func mixAddr(a string) (*common.MixedcaseAddress, error) { return common.NewMixedcaseAddressFromString(a) } func toHexBig(h string) hexutil.Big { - b := big.NewInt(0).SetBytes(common.FromHex(h)) + b := new(big.Int).SetBytes(common.FromHex(h)) return hexutil.Big(*b) } func toHexUint(h string) hexutil.Uint64 { - b := big.NewInt(0).SetBytes(common.FromHex(h)) + b := new(big.Int).SetBytes(common.FromHex(h)) return hexutil.Uint64(b.Uint64()) } func dummyTxArgs(t txtestcase) *apitypes.SendTxArgs { @@ -53,7 +53,6 @@ func dummyTxArgs(t txtestcase) *apitypes.SendTxArgs { if t.i != "" { a := hexutil.Bytes(common.FromHex(t.i)) input = &a - } return &apitypes.SendTxArgs{ From: *from, diff --git a/signer/rules/rules.go b/signer/rules/rules.go index 6852d86f3ec72..5ed4514e02279 100644 --- a/signer/rules/rules.go +++ b/signer/rules/rules.go @@ -59,6 +59,7 @@ func NewRuleEvaluator(next core.UIClientAPI, jsbackend storage.Storage) (*rulese return c, nil } func (r *rulesetUI) RegisterUIServer(api *core.UIServerAPI) { + r.next.RegisterUIServer(api) // TODO, make it possible to query from js } @@ -67,7 +68,6 @@ func (r *rulesetUI) Init(javascriptRules string) error { return nil } func (r *rulesetUI) execute(jsfunc string, jsarg interface{}) (goja.Value, error) { - // Instantiate a fresh vm engine every time vm := goja.New() diff --git a/signer/rules/rules_test.go b/signer/rules/rules_test.go index 0ab246eeaf7e7..c35da8ecc188c 100644 --- a/signer/rules/rules_test.go +++ b/signer/rules/rules_test.go @@ -44,7 +44,7 @@ Three things can happen: 3. Anything else; other return values [*], method not implemented or exception occurred during processing. This means that the operation will continue to manual processing, via the regular UI method chosen by the user. -[*] Note: Future version of the ruleset may use more complex json-based returnvalues, making it possible to not +[*] Note: Future version of the ruleset may use more complex json-based return values, making it possible to not only respond Approve/Reject/Manual, but also modify responses. For example, choose to list only one, but not all accounts in a list-request. The points above will continue to hold for non-json based responses ("Approve"/"Reject"). @@ -152,7 +152,6 @@ func TestListRequest(t *testing.T) { } func TestSignTxRequest(t *testing.T) { - js := ` function ApproveTx(r){ console.log("transaction.from", r.transaction.from); @@ -243,9 +242,8 @@ func (d *dummyUI) OnApprovedTx(tx ethapi.SignTransactionResult) { func (d *dummyUI) OnSignerStartup(info core.StartupInfo) { } -//TestForwarding tests that the rule-engine correctly dispatches requests to the next caller +// TestForwarding tests that the rule-engine correctly dispatches requests to the next caller func TestForwarding(t *testing.T) { - js := "" ui := &dummyUI{make([]string, 0)} jsBackend := storage.NewEphemeralStorage() @@ -268,11 +266,8 @@ func TestForwarding(t *testing.T) { expCalls := 6 if len(ui.calls) != expCalls { - t.Errorf("Expected %d forwarded calls, got %d: %s", expCalls, len(ui.calls), strings.Join(ui.calls, ",")) - } - } func TestMissingFunc(t *testing.T) { @@ -296,10 +291,8 @@ func TestMissingFunc(t *testing.T) { t.Errorf("Expected missing method to cause non-approval") } t.Logf("Err %v", err) - } func TestStorage(t *testing.T) { - js := ` function testStorage(){ storage.put("mykey", "myvalue") @@ -348,7 +341,6 @@ func TestStorage(t *testing.T) { t.Errorf("Unexpected data, expected '%v', got '%v'", exp, retval) } t.Logf("Err %v", err) - } const ExampleTxWindow = ` @@ -442,14 +434,14 @@ func dummyTx(value hexutil.Big) *core.SignTxRequest { Gas: gas, }, Callinfo: []apitypes.ValidationInfo{ - {Typ: "Warning", Message: "All your base are bellong to us"}, + {Typ: "Warning", Message: "All your base are belong to us"}, }, Meta: core.Metadata{Remote: "remoteip", Local: "localip", Scheme: "inproc"}, } } func dummyTxWithV(value uint64) *core.SignTxRequest { - v := big.NewInt(0).SetUint64(value) + v := new(big.Int).SetUint64(value) h := hexutil.Big(*v) return dummyTx(h) } @@ -469,7 +461,7 @@ func TestLimitWindow(t *testing.T) { return } // 0.3 ether: 429D069189E0000 wei - v := big.NewInt(0).SetBytes(common.Hex2Bytes("0429D069189E0000")) + v := new(big.Int).SetBytes(common.Hex2Bytes("0429D069189E0000")) h := hexutil.Big(*v) // The first three should succeed for i := 0; i < 3; i++ { @@ -544,11 +536,10 @@ func (d *dontCallMe) OnApprovedTx(tx ethapi.SignTransactionResult) { d.t.Fatalf("Did not expect next-handler to be called") } -//TestContextIsCleared tests that the rule-engine does not retain variables over several requests. +// TestContextIsCleared tests that the rule-engine does not retain variables over several requests. // if it does, that would be bad since developers may rely on that to store data, // instead of using the disk-based data storage func TestContextIsCleared(t *testing.T) { - js := ` function ApproveTx(){ if (typeof foobar == 'undefined') { @@ -580,7 +571,6 @@ func TestContextIsCleared(t *testing.T) { } func TestSignData(t *testing.T) { - js := `function ApproveListing(){ return "Approve" } diff --git a/signer/storage/aes_gcm_storage.go b/signer/storage/aes_gcm_storage.go index f09bfa7d4f06a..928d643dd618b 100644 --- a/signer/storage/aes_gcm_storage.go +++ b/signer/storage/aes_gcm_storage.go @@ -143,7 +143,7 @@ func (s *AESEncryptedStorage) writeEncryptedStorage(creds map[string]storedCrede // encrypt encrypts plaintext with the given key, with additional data // The 'additionalData' is used to place the (plaintext) KV-store key into the V, -// to prevent the possibility to alter a K, or swap two entries in the KV store with eachother. +// to prevent the possibility to alter a K, or swap two entries in the KV store with each other. func encrypt(key []byte, plaintext []byte, additionalData []byte) ([]byte, []byte, error) { block, err := aes.NewCipher(key) if err != nil { diff --git a/signer/storage/aes_gcm_storage_test.go b/signer/storage/aes_gcm_storage_test.go index a2a95d9deedf6..e1fea59280a82 100644 --- a/signer/storage/aes_gcm_storage_test.go +++ b/signer/storage/aes_gcm_storage_test.go @@ -51,7 +51,6 @@ func TestEncryption(t *testing.T) { } func TestFileStorage(t *testing.T) { - a := map[string]storedCredential{ "secret": { Iv: common.Hex2Bytes("cdb30036279601aeee60f16b"), diff --git a/tests/difficulty_test.go b/tests/difficulty_test.go index 192dff12cc97c..8239850b76177 100644 --- a/tests/difficulty_test.go +++ b/tests/difficulty_test.go @@ -79,6 +79,9 @@ func TestDifficulty(t *testing.T) { dt.config("EIP4345", params.ChainConfig{ ArrowGlacierBlock: big.NewInt(0), }) + dt.config("EIP5133", params.ChainConfig{ + GrayGlacierBlock: big.NewInt(0), + }) dt.config("difficulty.json", mainnetChainConfig) dt.walk(t, difficultyTestDir, func(t *testing.T, name string, test *DifficultyTest) { diff --git a/tests/difficulty_test_util.go b/tests/difficulty_test_util.go index bda5a9611be89..62b978f9ef2b0 100644 --- a/tests/difficulty_test_util.go +++ b/tests/difficulty_test_util.go @@ -65,5 +65,4 @@ func (test *DifficultyTest) Run(config *params.ChainConfig) error { test.CurrentTimestamp, test.CurrentBlockNumber, actual, exp) } return nil - } diff --git a/tests/fuzzers/bls12381/bls12381_fuzz.go b/tests/fuzzers/bls12381/bls12381_fuzz.go index b283ed11fe395..a5b4b9e798569 100644 --- a/tests/fuzzers/bls12381/bls12381_fuzz.go +++ b/tests/fuzzers/bls12381/bls12381_fuzz.go @@ -29,20 +29,22 @@ import ( gnark "github.com/consensys/gnark-crypto/ecc/bls12-381" "github.com/consensys/gnark-crypto/ecc/bls12-381/fp" "github.com/consensys/gnark-crypto/ecc/bls12-381/fr" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto/bls12381" + blst "github.com/supranational/blst/bindings/go" ) func FuzzCrossPairing(data []byte) int { input := bytes.NewReader(data) // get random G1 points - kpG1, cpG1, err := getG1Points(input) + kpG1, cpG1, blG1, err := getG1Points(input) if err != nil { return 0 } // get random G2 points - kpG2, cpG2, err := getG2Points(input) + kpG2, cpG2, blG2, err := getG2Points(input) if err != nil { return 0 } @@ -63,20 +65,52 @@ func FuzzCrossPairing(data []byte) int { panic("pairing mismatch gnark / geth ") } + // compute pairing using blst + blstResult := blst.Fp12MillerLoop(blG2, blG1) + blstResult.FinalExp() + res := massageBLST(blstResult.ToBendian()) + if !(bytes.Equal(res, bls12381.NewGT().ToBytes(kResult))) { + panic("pairing mismatch blst / geth") + } + return 1 } +func massageBLST(in []byte) []byte { + out := make([]byte, len(in)) + len := 12 * 48 + // 1 + copy(out[0:], in[len-1*48:len]) + copy(out[1*48:], in[len-2*48:len-1*48]) + // 2 + copy(out[6*48:], in[len-3*48:len-2*48]) + copy(out[7*48:], in[len-4*48:len-3*48]) + // 3 + copy(out[2*48:], in[len-5*48:len-4*48]) + copy(out[3*48:], in[len-6*48:len-5*48]) + // 4 + copy(out[8*48:], in[len-7*48:len-6*48]) + copy(out[9*48:], in[len-8*48:len-7*48]) + // 5 + copy(out[4*48:], in[len-9*48:len-8*48]) + copy(out[5*48:], in[len-10*48:len-9*48]) + // 6 + copy(out[10*48:], in[len-11*48:len-10*48]) + copy(out[11*48:], in[len-12*48:len-11*48]) + return out +} + func FuzzCrossG1Add(data []byte) int { input := bytes.NewReader(data) // get random G1 points - kp1, cp1, err := getG1Points(input) + kp1, cp1, bl1, err := getG1Points(input) if err != nil { return 0 } // get random G1 points - kp2, cp2, err := getG1Points(input) + kp2, cp2, bl2, err := getG1Points(input) if err != nil { return 0 } @@ -96,6 +130,11 @@ func FuzzCrossG1Add(data []byte) int { panic("G1 point addition mismatch gnark / geth ") } + bl3 := blst.P1AffinesAdd([]*blst.P1Affine{bl1, bl2}) + if !(bytes.Equal(cp.Marshal(), bl3.Serialize())) { + panic("G1 point addition mismatch blst / geth ") + } + return 1 } @@ -103,13 +142,13 @@ func FuzzCrossG2Add(data []byte) int { input := bytes.NewReader(data) // get random G2 points - kp1, cp1, err := getG2Points(input) + kp1, cp1, bl1, err := getG2Points(input) if err != nil { return 0 } // get random G2 points - kp2, cp2, err := getG2Points(input) + kp2, cp2, bl2, err := getG2Points(input) if err != nil { return 0 } @@ -129,6 +168,11 @@ func FuzzCrossG2Add(data []byte) int { panic("G2 point addition mismatch gnark / geth ") } + bl3 := blst.P2AffinesAdd([]*blst.P2Affine{bl1, bl2}) + if !(bytes.Equal(cp.Marshal(), bl3.Serialize())) { + panic("G1 point addition mismatch blst / geth ") + } + return 1 } @@ -148,7 +192,7 @@ func FuzzCrossG1MultiExp(data []byte) int { break } // get a random G1 point as basis - kp1, cp1, err := getG1Points(input) + kp1, cp1, _, err := getG1Points(input) if err != nil { break } @@ -183,11 +227,11 @@ func FuzzCrossG1MultiExp(data []byte) int { return 1 } -func getG1Points(input io.Reader) (*bls12381.PointG1, *gnark.G1Affine, error) { +func getG1Points(input io.Reader) (*bls12381.PointG1, *gnark.G1Affine, *blst.P1Affine, error) { // sample a random scalar s, err := randomScalar(input, fp.Modulus()) if err != nil { - return nil, nil, err + return nil, nil, nil, err } // compute a random point @@ -206,14 +250,21 @@ func getG1Points(input io.Reader) (*bls12381.PointG1, *gnark.G1Affine, error) { panic("bytes(gnark.G1) != bytes(geth.G1)") } - return kp, cp, nil + // marshal gnark point -> blst point + scalar := new(blst.Scalar).FromBEndian(common.LeftPadBytes(s.Bytes(), 32)) + p1 := new(blst.P1Affine).From(scalar) + if !bytes.Equal(p1.Serialize(), cpBytes) { + panic("bytes(blst.G1) != bytes(geth.G1)") + } + + return kp, cp, p1, nil } -func getG2Points(input io.Reader) (*bls12381.PointG2, *gnark.G2Affine, error) { +func getG2Points(input io.Reader) (*bls12381.PointG2, *gnark.G2Affine, *blst.P2Affine, error) { // sample a random scalar s, err := randomScalar(input, fp.Modulus()) if err != nil { - return nil, nil, err + return nil, nil, nil, err } // compute a random point @@ -232,7 +283,15 @@ func getG2Points(input io.Reader) (*bls12381.PointG2, *gnark.G2Affine, error) { panic("bytes(gnark.G2) != bytes(geth.G2)") } - return kp, cp, nil + // marshal gnark point -> blst point + // Left pad the scalar to 32 bytes + scalar := new(blst.Scalar).FromBEndian(common.LeftPadBytes(s.Bytes(), 32)) + p2 := new(blst.P2Affine).From(scalar) + if !bytes.Equal(p2.Serialize(), cpBytes) { + panic("bytes(blst.G2) != bytes(geth.G2)") + } + + return kp, cp, p2, nil } func randomScalar(r io.Reader, max *big.Int) (k *big.Int, err error) { diff --git a/tests/fuzzers/difficulty/difficulty-fuzz.go b/tests/fuzzers/difficulty/difficulty-fuzz.go index 58936fcd80b10..2112abac1afa3 100644 --- a/tests/fuzzers/difficulty/difficulty-fuzz.go +++ b/tests/fuzzers/difficulty/difficulty-fuzz.go @@ -30,7 +30,6 @@ import ( type fuzzer struct { input io.Reader exhausted bool - debugging bool } func (f *fuzzer) read(size int) []byte { diff --git a/tests/fuzzers/les/les-fuzzer.go b/tests/fuzzers/les/les-fuzzer.go index 3e10171873452..677702208871c 100644 --- a/tests/fuzzers/les/les-fuzzer.go +++ b/tests/fuzzers/les/les-fuzzer.go @@ -88,8 +88,8 @@ func makechain() (bc *core.BlockChain, addrHashes, txHashes []common.Hash) { } func makeTries() (chtTrie *trie.Trie, bloomTrie *trie.Trie, chtKeys, bloomKeys [][]byte) { - chtTrie, _ = trie.New(common.Hash{}, trie.NewDatabase(rawdb.NewMemoryDatabase())) - bloomTrie, _ = trie.New(common.Hash{}, trie.NewDatabase(rawdb.NewMemoryDatabase())) + chtTrie = trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase())) + bloomTrie = trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase())) for i := 0; i < testChainLen; i++ { // The element in CHT is -> key := make([]byte, 8) diff --git a/tests/fuzzers/rangeproof/rangeproof-fuzzer.go b/tests/fuzzers/rangeproof/rangeproof-fuzzer.go index 18717e70d0012..c2db919d5a978 100644 --- a/tests/fuzzers/rangeproof/rangeproof-fuzzer.go +++ b/tests/fuzzers/rangeproof/rangeproof-fuzzer.go @@ -62,7 +62,7 @@ func (f *fuzzer) readInt() uint64 { } func (f *fuzzer) randomTrie(n int) (*trie.Trie, map[string]*kv) { - trie, _ := trie.New(common.Hash{}, trie.NewDatabase(rawdb.NewMemoryDatabase())) + trie := trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase())) vals := make(map[string]*kv) size := f.readInt() // Fill it with some fluff @@ -163,7 +163,6 @@ func (f *fuzzer) fuzz() int { // Modify something in the proof db // add stuff to proof db // drop stuff from proof db - } if f.exhausted { break diff --git a/tests/fuzzers/rlp/rlp_fuzzer.go b/tests/fuzzers/rlp/rlp_fuzzer.go index 18b36287b53c2..ac02e1651d446 100644 --- a/tests/fuzzers/rlp/rlp_fuzzer.go +++ b/tests/fuzzers/rlp/rlp_fuzzer.go @@ -40,6 +40,9 @@ func Fuzz(input []byte) int { if len(input) == 0 { return 0 } + if len(input) > 500*1024 { + return 0 + } var i int { diff --git a/tests/fuzzers/stacktrie/trie_fuzzer.go b/tests/fuzzers/stacktrie/trie_fuzzer.go index 9ed8bcbc51d58..e6165df08c156 100644 --- a/tests/fuzzers/stacktrie/trie_fuzzer.go +++ b/tests/fuzzers/stacktrie/trie_fuzzer.go @@ -25,7 +25,6 @@ import ( "io" "sort" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/trie" "golang.org/x/crypto/sha3" @@ -139,12 +138,11 @@ func Debug(data []byte) int { } func (f *fuzzer) fuzz() int { - // This spongeDb is used to check the sequence of disk-db-writes var ( spongeA = &spongeDb{sponge: sha3.NewLegacyKeccak256()} dbA = trie.NewDatabase(spongeA) - trieA, _ = trie.New(common.Hash{}, dbA) + trieA = trie.NewEmpty(dbA) spongeB = &spongeDb{sponge: sha3.NewLegacyKeccak256()} trieB = trie.NewStackTrie(spongeB) vals kvs @@ -175,10 +173,13 @@ func (f *fuzzer) fuzz() int { return 0 } // Flush trie -> database - rootA, _, err := trieA.Commit(nil) + rootA, nodes, err := trieA.Commit(false) if err != nil { panic(err) } + if nodes != nil { + dbA.Update(trie.NewWithNodeSet(nodes)) + } // Flush memdb -> disk (sponge) dbA.Commit(rootA, false, nil) @@ -186,7 +187,7 @@ func (f *fuzzer) fuzz() int { sort.Sort(vals) for _, kv := range vals { if f.debugging { - fmt.Printf("{\"0x%x\" , \"0x%x\"} // stacktrie.Update\n", kv.k, kv.v) + fmt.Printf("{\"%#x\" , \"%#x\"} // stacktrie.Update\n", kv.k, kv.v) } trieB.Update(kv.k, kv.v) } diff --git a/tests/fuzzers/trie/trie-fuzzer.go b/tests/fuzzers/trie/trie-fuzzer.go index e993af47cf20c..f36b613d47863 100644 --- a/tests/fuzzers/trie/trie-fuzzer.go +++ b/tests/fuzzers/trie/trie-fuzzer.go @@ -51,9 +51,8 @@ const ( opUpdate = iota opDelete opGet - opCommit opHash - opReset + opCommit opItercheckhash opProve opMax // boundary value, not an actual op @@ -84,11 +83,9 @@ func (ds *dataSource) Ended() bool { } func Generate(input []byte) randTest { - var allKeys [][]byte r := newDataSource(input) genKey := func() []byte { - if len(allKeys) < 2 || r.readByte() < 0x0f { // new key key := make([]byte, r.readByte()%50) @@ -103,7 +100,6 @@ func Generate(input []byte) randTest { var steps randTest for i := 0; !r.Ended(); i++ { - step := randTestStep{op: int(r.readByte()) % opMax} switch step.op { case opUpdate: @@ -141,10 +137,9 @@ func Fuzz(input []byte) int { } func runRandTest(rt randTest) error { - triedb := trie.NewDatabase(memorydb.New()) - tr, _ := trie.New(common.Hash{}, triedb) + tr := trie.NewEmpty(triedb) values := make(map[string]string) // tracks content of the trie for i, step := range rt { @@ -159,24 +154,27 @@ func runRandTest(rt randTest) error { v := tr.Get(step.key) want := values[string(step.key)] if string(v) != want { - rt[i].err = fmt.Errorf("mismatch for key 0x%x, got 0x%x want 0x%x", step.key, v, want) + rt[i].err = fmt.Errorf("mismatch for key %#x, got %#x want %#x", step.key, v, want) } - case opCommit: - _, _, rt[i].err = tr.Commit(nil) case opHash: tr.Hash() - case opReset: - hash, _, err := tr.Commit(nil) + case opCommit: + hash, nodes, err := tr.Commit(false) if err != nil { return err } - newtr, err := trie.New(hash, triedb) + if nodes != nil { + if err := triedb.Update(trie.NewWithNodeSet(nodes)); err != nil { + return err + } + } + newtr, err := trie.New(common.Hash{}, hash, triedb) if err != nil { return err } tr = newtr case opItercheckhash: - checktr, _ := trie.New(common.Hash{}, triedb) + checktr := trie.NewEmpty(triedb) it := trie.NewIterator(tr.NodeIterator(nil)) for it.Next() { checktr.Update(it.Key, it.Value) diff --git a/tests/init.go b/tests/init.go index 52277e8416423..87ffc65a677c9 100644 --- a/tests/init.go +++ b/tests/init.go @@ -197,6 +197,22 @@ var Forks = map[string]*params.ChainConfig{ LondonBlock: big.NewInt(0), ArrowGlacierBlock: big.NewInt(0), }, + "GrayGlacier": { + ChainID: big.NewInt(1), + HomesteadBlock: big.NewInt(0), + EIP150Block: big.NewInt(0), + EIP155Block: big.NewInt(0), + EIP158Block: big.NewInt(0), + ByzantiumBlock: big.NewInt(0), + ConstantinopleBlock: big.NewInt(0), + PetersburgBlock: big.NewInt(0), + IstanbulBlock: big.NewInt(0), + MuirGlacierBlock: big.NewInt(0), + BerlinBlock: big.NewInt(0), + LondonBlock: big.NewInt(0), + ArrowGlacierBlock: big.NewInt(0), + GrayGlacierBlock: big.NewInt(0), + }, "Merged": { ChainID: big.NewInt(1), HomesteadBlock: big.NewInt(0), @@ -211,7 +227,7 @@ var Forks = map[string]*params.ChainConfig{ BerlinBlock: big.NewInt(0), LondonBlock: big.NewInt(0), ArrowGlacierBlock: big.NewInt(0), - MergeForkBlock: big.NewInt(0), + MergeNetsplitBlock: big.NewInt(0), TerminalTotalDifficulty: big.NewInt(0), }, } diff --git a/tests/init_test.go b/tests/init_test.go index 218634966d83f..4ef5aaf7378fc 100644 --- a/tests/init_test.go +++ b/tests/init_test.go @@ -116,6 +116,7 @@ func (tm *testMatcher) skipLoad(pattern string) { } // fails adds an expected failure for tests matching the pattern. +//nolint:unused func (tm *testMatcher) fails(pattern string, reason string) { if reason == "" { panic("empty fail reason") diff --git a/tests/rlp_test_util.go b/tests/rlp_test_util.go index 9069ec55a15dc..15acb3a244f77 100644 --- a/tests/rlp_test_util.go +++ b/tests/rlp_test_util.go @@ -124,7 +124,7 @@ func translateJSON(v interface{}) interface{} { func checkDecodeFromJSON(s *rlp.Stream, exp interface{}) error { switch exp := exp.(type) { case uint64: - i, err := s.Uint() + i, err := s.Uint64() if err != nil { return addStack("Uint", exp, err) } diff --git a/tests/state_test.go b/tests/state_test.go index d2c92b211cd17..d33ebc4b00db5 100644 --- a/tests/state_test.go +++ b/tests/state_test.go @@ -174,6 +174,7 @@ func runBenchmarkFile(b *testing.B, path string) { return } for _, t := range m { + t := t runBenchmark(b, &t) } } @@ -192,7 +193,7 @@ func runBenchmark(b *testing.B, t *StateTest) { return } vmconfig.ExtraEips = eips - block := t.genesis(config).ToBlock(nil) + block := t.genesis(config).ToBlock() _, statedb := MakePreState(rawdb.NewMemoryDatabase(), t.json.Pre, false) var baseFee *big.Int @@ -248,7 +249,6 @@ func runBenchmark(b *testing.B, t *StateTest) { } statedb.RevertToSnapshot(snapshot) } - }) } } diff --git a/tests/state_test_util.go b/tests/state_test_util.go index f6d8e15001d87..38cdbc4d65043 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -184,7 +184,7 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh return nil, nil, common.Hash{}, UnsupportedForkError{subtest.Fork} } vmconfig.ExtraEips = eips - block := t.genesis(config).ToBlock(nil) + block := t.genesis(config).ToBlock() snaps, statedb := MakePreState(rawdb.NewMemoryDatabase(), t.json.Pre, snapshotter) var baseFee *big.Int @@ -220,7 +220,8 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh context := core.NewEVMBlockContext(block.Header(), nil, &t.json.Env.Coinbase) context.GetHash = vmTestBlockHash context.BaseFee = baseFee - if t.json.Env.Random != nil { + context.Random = nil + if config.IsLondon(new(big.Int)) && t.json.Env.Random != nil { rnd := common.BigToHash(t.json.Env.Random) context.Random = &rnd context.Difficulty = big.NewInt(0) diff --git a/trie/committer.go b/trie/committer.go index 9a7bf48d977fd..d9f0ecf3dea41 100644 --- a/trie/committer.go +++ b/trie/committer.go @@ -17,71 +17,48 @@ package trie import ( - "errors" "fmt" - "sync" "github.com/ethereum/go-ethereum/common" ) -// leafChanSize is the size of the leafCh. It's a pretty arbitrary number, to allow -// some parallelism but not incur too much memory overhead. -const leafChanSize = 200 - -// leaf represents a trie leaf value +// leaf represents a trie leaf node type leaf struct { - size int // size of the rlp data (estimate) - hash common.Hash // hash of rlp data - node node // the node to commit + blob []byte // raw blob of leaf + parent common.Hash // the hash of parent node } -// committer is a type used for the trie Commit operation. A committer has some -// internal preallocated temp space, and also a callback that is invoked when -// leaves are committed. The leafs are passed through the `leafCh`, to allow -// some level of parallelism. -// By 'some level' of parallelism, it's still the case that all leaves will be -// processed sequentially - onleaf will never be called in parallel or out of order. +// committer is the tool used for the trie Commit operation. The committer will +// capture all dirty nodes during the commit process and keep them cached in +// insertion order. type committer struct { - onleaf LeafCallback - leafCh chan *leaf -} - -// committers live in a global sync.Pool -var committerPool = sync.Pool{ - New: func() interface{} { - return &committer{} - }, + nodes *NodeSet + collectLeaf bool } // newCommitter creates a new committer or picks one from the pool. -func newCommitter() *committer { - return committerPool.Get().(*committer) -} - -func returnCommitterToPool(h *committer) { - h.onleaf = nil - h.leafCh = nil - committerPool.Put(h) +func newCommitter(owner common.Hash, collectLeaf bool) *committer { + return &committer{ + nodes: NewNodeSet(owner), + collectLeaf: collectLeaf, + } } // Commit collapses a node down into a hash node and inserts it into the database -func (c *committer) Commit(n node, db *Database) (hashNode, int, error) { - if db == nil { - return nil, 0, errors.New("no db provided") - } - h, committed, err := c.commit(n, db) +func (c *committer) Commit(n node) (hashNode, *NodeSet, error) { + h, err := c.commit(nil, n) if err != nil { - return nil, 0, err + return nil, nil, err } - return h.(hashNode), committed, nil + return h.(hashNode), c.nodes, nil } // commit collapses a node down into a hash node and inserts it into the database -func (c *committer) commit(n node, db *Database) (node, int, error) { +func (c *committer) commit(path []byte, n node) (node, error) { // if this path is clean, use available cached data hash, dirty := n.cache() if hash != nil && !dirty { - return hash, 0, nil + return hash, nil } // Commit children, then parent, and remove the dirty flag. switch cn := n.(type) { @@ -91,36 +68,35 @@ func (c *committer) commit(n node, db *Database) (node, int, error) { // If the child is fullNode, recursively commit, // otherwise it can only be hashNode or valueNode. - var childCommitted int if _, ok := cn.Val.(*fullNode); ok { - childV, committed, err := c.commit(cn.Val, db) + childV, err := c.commit(append(path, cn.Key...), cn.Val) if err != nil { - return nil, 0, err + return nil, err } - collapsed.Val, childCommitted = childV, committed + collapsed.Val = childV } // The key needs to be copied, since we're delivering it to database collapsed.Key = hexToCompact(cn.Key) - hashedNode := c.store(collapsed, db) + hashedNode := c.store(path, collapsed) if hn, ok := hashedNode.(hashNode); ok { - return hn, childCommitted + 1, nil + return hn, nil } - return collapsed, childCommitted, nil + return collapsed, nil case *fullNode: - hashedKids, childCommitted, err := c.commitChildren(cn, db) + hashedKids, err := c.commitChildren(path, cn) if err != nil { - return nil, 0, err + return nil, err } collapsed := cn.copy() collapsed.Children = hashedKids - hashedNode := c.store(collapsed, db) + hashedNode := c.store(path, collapsed) if hn, ok := hashedNode.(hashNode); ok { - return hn, childCommitted + 1, nil + return hn, nil } - return collapsed, childCommitted, nil + return collapsed, nil case hashNode: - return cn, 0, nil + return cn, nil default: // nil, valuenode shouldn't be committed panic(fmt.Sprintf("%T: invalid node: %v", n, n)) @@ -128,11 +104,8 @@ func (c *committer) commit(n node, db *Database) (node, int, error) { } // commitChildren commits the children of the given fullnode -func (c *committer) commitChildren(n *fullNode, db *Database) ([17]node, int, error) { - var ( - committed int - children [17]node - ) +func (c *committer) commitChildren(path []byte, n *fullNode) ([17]node, error) { + var children [17]node for i := 0; i < 16; i++ { child := n.Children[i] if child == nil { @@ -148,91 +121,63 @@ func (c *committer) commitChildren(n *fullNode, db *Database) ([17]node, int, er // Commit the child recursively and store the "hashed" value. // Note the returned node can be some embedded nodes, so it's // possible the type is not hashNode. - hashed, childCommitted, err := c.commit(child, db) + hashed, err := c.commit(append(path, byte(i)), child) if err != nil { - return children, 0, err + return children, err } children[i] = hashed - committed += childCommitted } // For the 17th child, it's possible the type is valuenode. if n.Children[16] != nil { children[16] = n.Children[16] } - return children, committed, nil + return children, nil } // store hashes the node n and if we have a storage layer specified, it writes // the key/value pair to it and tracks any node->child references as well as any // node->external trie references. -func (c *committer) store(n node, db *Database) node { +func (c *committer) store(path []byte, n node) node { // Larger nodes are replaced by their hash and stored in the database. - var ( - hash, _ = n.cache() - size int - ) + var hash, _ = n.cache() + + // This was not generated - must be a small node stored in the parent. + // In theory, we should check if the node is leaf here (embedded node + // usually is leaf node). But small value(less than 32bytes) is not + // our target(leaves in account trie only). if hash == nil { - // This was not generated - must be a small node stored in the parent. - // In theory, we should apply the leafCall here if it's not nil(embedded - // node usually contains value). But small value(less than 32bytes) is - // not our target. return n - } else { - // We have the hash already, estimate the RLP encoding-size of the node. - // The size is used for mem tracking, does not need to be exact - size = estimateSize(n) } - // If we're using channel-based leaf-reporting, send to channel. - // The leaf channel will be active only when there an active leaf-callback - if c.leafCh != nil { - c.leafCh <- &leaf{ - size: size, - hash: common.BytesToHash(hash), - node: n, + // We have the hash already, estimate the RLP encoding-size of the node. + // The size is used for mem tracking, does not need to be exact + var ( + size = estimateSize(n) + nhash = common.BytesToHash(hash) + mnode = &memoryNode{ + hash: nhash, + node: simplifyNode(n), + size: uint16(size), } - } else if db != nil { - // No leaf-callback used, but there's still a database. Do serial - // insertion - db.lock.Lock() - db.insert(common.BytesToHash(hash), size, n) - db.lock.Unlock() - } - return hash -} - -// commitLoop does the actual insert + leaf callback for nodes. -func (c *committer) commitLoop(db *Database) { - for item := range c.leafCh { - var ( - hash = item.hash - size = item.size - n = item.node - ) - // We are pooling the trie nodes into an intermediate memory cache - db.lock.Lock() - db.insert(hash, size, n) - db.lock.Unlock() - - if c.onleaf != nil { - switch n := n.(type) { - case *shortNode: - if child, ok := n.Val.(valueNode); ok { - c.onleaf(nil, nil, child, hash) - } - case *fullNode: - // For children in range [0, 15], it's impossible - // to contain valueNode. Only check the 17th child. - if n.Children[16] != nil { - c.onleaf(nil, nil, n.Children[16].(valueNode), hash) - } + ) + // Collect the dirty node to nodeset for return. + c.nodes.add(string(path), mnode) + + // Collect the corresponding leaf node if it's required. We don't check + // full node since it's impossible to store value in fullNode. The key + // length of leaves should be exactly same. + if c.collectLeaf { + if sn, ok := n.(*shortNode); ok { + if val, ok := sn.Val.(valueNode); ok { + c.nodes.addLeaf(&leaf{blob: val, parent: nhash}) } } } + return hash } // estimateSize estimates the size of an rlp-encoded node, without actually // rlp-encoding it (zero allocs). This method has been experimentally tried, and with a trie -// with 1000 leafs, the only errors above 1% are on small shortnodes, where this +// with 1000 leaves, the only errors above 1% are on small shortnodes, where this // method overestimates by 2 or 3 bytes (e.g. 37 instead of 35) func estimateSize(n node) int { switch n := n.(type) { diff --git a/trie/database.go b/trie/database.go index d71abeee476a5..b10bbca9bdbaa 100644 --- a/trie/database.go +++ b/trie/database.go @@ -28,6 +28,7 @@ import ( "github.com/VictoriaMetrics/fastcache" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" @@ -74,8 +75,6 @@ type Database struct { oldest common.Hash // Oldest tracked node, flush-list head newest common.Hash // Newest tracked node, flush-list tail - preimages map[common.Hash][]byte // Preimages of nodes from the secure trie - gctime time.Duration // Time spent on garbage collection since last commit gcnodes uint64 // Nodes garbage collected since last commit gcsize common.StorageSize // Data storage garbage collected since last commit @@ -84,9 +83,9 @@ type Database struct { flushnodes uint64 // Nodes flushed since last commit flushsize common.StorageSize // Data storage flushed since last commit - dirtiesSize common.StorageSize // Storage size of the dirty node cache (exc. metadata) - childrenSize common.StorageSize // Storage size of the external children tracking - preimagesSize common.StorageSize // Storage size of the preimages cache + dirtiesSize common.StorageSize // Storage size of the dirty node cache (exc. metadata) + childrenSize common.StorageSize // Storage size of the external children tracking + preimages *preimageStore // The store for caching preimages lock sync.RWMutex } @@ -164,7 +163,10 @@ func (n *cachedNode) rlp() []byte { // or by regenerating it from the rlp encoded blob. func (n *cachedNode) obj(hash common.Hash) node { if node, ok := n.node.(rawNode); ok { - return mustDecodeNode(hash[:], node) + // The raw-blob format nodes are loaded from either from + // clean cache or the database, they are all in their own + // copy and safe to use unsafe decoder. + return mustDecodeNodeUnsafe(hash[:], node) } return expandNode(hash[:], n.node) } @@ -287,15 +289,17 @@ func NewDatabaseWithConfig(diskdb ethdb.KeyValueStore, config *Config) *Database cleans = fastcache.LoadFromFileOrNew(config.Journal, config.Cache*1024*1024) } } + var preimage *preimageStore + if config != nil && config.Preimages { + preimage = newPreimageStore(diskdb) + } db := &Database{ diskdb: diskdb, cleans: cleans, dirties: map[common.Hash]*cachedNode{{}: { children: make(map[common.Hash]uint16), }}, - } - if config == nil || config.Preimages { // TODO(karalabe): Flip to default off in the future - db.preimages = make(map[common.Hash][]byte) + preimages: preimage, } return db } @@ -305,8 +309,7 @@ func (db *Database) DiskDB() ethdb.KeyValueStore { return db.diskdb } -// insert inserts a collapsed trie node into the memory database. -// The blob size must be specified to allow proper size tracking. +// insert inserts a simplified trie node into the memory database. // All nodes inserted by this function will be reference tracked // and in theory should only used for **trie nodes** insertion. func (db *Database) insert(hash common.Hash, size int, node node) { @@ -318,7 +321,7 @@ func (db *Database) insert(hash common.Hash, size int, node node) { // Create the cached entry for this node entry := &cachedNode{ - node: simplifyNode(node), + node: node, size: uint16(size), flushPrev: db.newest, } @@ -338,24 +341,6 @@ func (db *Database) insert(hash common.Hash, size int, node node) { db.dirtiesSize += common.StorageSize(common.HashLength + entry.size) } -// insertPreimage writes a new trie node pre-image to the memory database if it's -// yet unknown. The method will NOT make a copy of the slice, -// only use if the preimage will NOT be changed later on. -// -// Note, this method assumes that the database's lock is held! -func (db *Database) insertPreimage(hash common.Hash, preimage []byte) { - // Short circuit if preimage collection is disabled - if db.preimages == nil { - return - } - // Track the preimage if a yet unknown one - if _, ok := db.preimages[hash]; ok { - return - } - db.preimages[hash] = preimage - db.preimagesSize += common.StorageSize(common.HashLength + len(preimage)) -} - // node retrieves a cached trie node from memory, or returns nil if none can be // found in the memory cache. func (db *Database) node(hash common.Hash) node { @@ -364,7 +349,10 @@ func (db *Database) node(hash common.Hash) node { if enc := db.cleans.Get(nil, hash[:]); enc != nil { memcacheCleanHitMeter.Mark(1) memcacheCleanReadMeter.Mark(int64(len(enc))) - return mustDecodeNode(hash[:], enc) + + // The returned value from cache is in its own copy, + // safe to use mustDecodeNodeUnsafe for decoding. + return mustDecodeNodeUnsafe(hash[:], enc) } } // Retrieve the node from the dirty cache if available @@ -389,7 +377,9 @@ func (db *Database) node(hash common.Hash) node { memcacheCleanMissMeter.Mark(1) memcacheCleanWriteMeter.Mark(int64(len(enc))) } - return mustDecodeNode(hash[:], enc) + // The returned value from database is in its own copy, + // safe to use mustDecodeNodeUnsafe for decoding. + return mustDecodeNodeUnsafe(hash[:], enc) } // Node retrieves an encoded cached trie node from memory. If it cannot be found @@ -432,24 +422,6 @@ func (db *Database) Node(hash common.Hash) ([]byte, error) { return nil, errors.New("not found") } -// preimage retrieves a cached trie node pre-image from memory. If it cannot be -// found cached, the method queries the persistent database for the content. -func (db *Database) preimage(hash common.Hash) []byte { - // Short circuit if preimage collection is disabled - if db.preimages == nil { - return nil - } - // Retrieve the node from cache if available - db.lock.RLock() - preimage := db.preimages[hash] - db.lock.RUnlock() - - if preimage != nil { - return preimage - } - return rawdb.ReadPreimage(db.diskdb, hash) -} - // Nodes retrieves the hashes of all the nodes cached within the memory database. // This method is extremely expensive and should only be used to validate internal // states in test code. @@ -594,19 +566,8 @@ func (db *Database) Cap(limit common.StorageSize) error { // If the preimage cache got large enough, push to disk. If it's still small // leave for later to deduplicate writes. - flushPreimages := db.preimagesSize > 4*1024*1024 - if flushPreimages { - if db.preimages == nil { - log.Error("Attempted to write preimages whilst disabled") - } else { - rawdb.WritePreimages(batch, db.preimages) - if batch.ValueSize() > ethdb.IdealBatchSize { - if err := batch.Write(); err != nil { - return err - } - batch.Reset() - } - } + if db.preimages != nil { + db.preimages.commit(false) } // Keep committing nodes from the flush-list until we're below allowance oldest := db.oldest @@ -641,13 +602,6 @@ func (db *Database) Cap(limit common.StorageSize) error { db.lock.Lock() defer db.lock.Unlock() - if flushPreimages { - if db.preimages == nil { - log.Error("Attempted to reset preimage cache whilst disabled") - } else { - db.preimages, db.preimagesSize = make(map[common.Hash][]byte), 0 - } - } for db.oldest != oldest { node := db.dirties[db.oldest] delete(db.dirties, db.oldest) @@ -691,13 +645,7 @@ func (db *Database) Commit(node common.Hash, report bool, callback func(common.H // Move all of the accumulated preimages into a write batch if db.preimages != nil { - rawdb.WritePreimages(batch, db.preimages) - // Since we're going to replay trie node writes into the clean cache, flush out - // any batched pre-images before continuing. - if err := batch.Write(); err != nil { - return err - } - batch.Reset() + db.preimages.commit(true) } // Move the trie itself into the batch, flushing if enough data is accumulated nodes, storage := len(db.dirties), db.dirtiesSize @@ -720,9 +668,6 @@ func (db *Database) Commit(node common.Hash, report bool, callback func(common.H batch.Reset() // Reset the storage counters and bumped metrics - if db.preimages != nil { - db.preimages, db.preimagesSize = make(map[common.Hash][]byte), 0 - } memcacheCommitTimeTimer.Update(time.Since(start)) memcacheCommitSizeMeter.Mark(int64(storage - db.dirtiesSize)) memcacheCommitNodesMeter.Mark(int64(nodes - len(db.dirties))) @@ -809,7 +754,7 @@ func (c *cleaner) Put(key []byte, rlp []byte) error { delete(c.db.dirties, hash) c.db.dirtiesSize -= common.StorageSize(common.HashLength + int(node.size)) if node.children != nil { - c.db.dirtiesSize -= common.StorageSize(cachedNodeChildrenSize + len(node.children)*(common.HashLength+2)) + c.db.childrenSize -= common.StorageSize(cachedNodeChildrenSize + len(node.children)*(common.HashLength+2)) } // Move the flushed node into the clean cache to prevent insta-reloads if c.db.cleans != nil { @@ -823,6 +768,54 @@ func (c *cleaner) Delete(key []byte) error { panic("not implemented") } +// Update inserts the dirty nodes in provided nodeset into database and +// link the account trie with multiple storage tries if necessary. +func (db *Database) Update(nodes *MergedNodeSet) error { + db.lock.Lock() + defer db.lock.Unlock() + + // Insert dirty nodes into the database. In the same tree, it must be + // ensured that children are inserted first, then parent so that children + // can be linked with their parent correctly. + // + // Note, the storage tries must be flushed before the account trie to + // retain the invariant that children go into the dirty cache first. + var order []common.Hash + for owner := range nodes.sets { + if owner == (common.Hash{}) { + continue + } + order = append(order, owner) + } + if _, ok := nodes.sets[common.Hash{}]; ok { + order = append(order, common.Hash{}) + } + for _, owner := range order { + subset := nodes.sets[owner] + for _, path := range subset.paths { + n, ok := subset.nodes[path] + if !ok { + return fmt.Errorf("missing node %x %v", owner, path) + } + db.insert(n.hash, int(n.size), n.node) + } + } + // Link up the account trie and storage trie if the node points + // to an account trie leaf. + if set, present := nodes.sets[common.Hash{}]; present { + for _, n := range set.leaves { + var account types.StateAccount + if err := rlp.DecodeBytes(n.blob, &account); err != nil { + return err + } + if account.Root != emptyRoot { + db.reference(account.Root, n.parent) + } + } + } + return nil +} + // Size returns the current storage size of the memory cache in front of the // persistent database layer. func (db *Database) Size() (common.StorageSize, common.StorageSize) { @@ -834,7 +827,11 @@ func (db *Database) Size() (common.StorageSize, common.StorageSize) { // counted. var metadataSize = common.StorageSize((len(db.dirties) - 1) * cachedNodeSize) var metarootRefs = common.StorageSize(len(db.dirties[common.Hash{}].children) * (common.HashLength + 2)) - return db.dirtiesSize + db.childrenSize + metadataSize - metarootRefs, db.preimagesSize + var preimageSize common.StorageSize + if db.preimages != nil { + preimageSize = db.preimages.size() + } + return db.dirtiesSize + db.childrenSize + metadataSize - metarootRefs, preimageSize } // saveCache saves clean state cache to given directory path @@ -876,3 +873,16 @@ func (db *Database) SaveCachePeriodically(dir string, interval time.Duration, st } } } + +// CommitPreimages flushes the dangling preimages to disk. It is meant to be +// called when closing the blockchain object, so that preimages are persisted +// to the database. +func (db *Database) CommitPreimages() error { + db.lock.Lock() + defer db.lock.Unlock() + + if db.preimages == nil { + return nil + } + return db.preimages.commit(true) +} diff --git a/trie/errors.go b/trie/errors.go index 567b80078c062..afe344bed2692 100644 --- a/trie/errors.go +++ b/trie/errors.go @@ -26,10 +26,21 @@ import ( // in the case where a trie node is not present in the local database. It contains // information necessary for retrieving the missing node. type MissingNodeError struct { + Owner common.Hash // owner of the trie if it's 2-layered trie NodeHash common.Hash // hash of the missing node Path []byte // hex-encoded path to the missing node + err error // concrete error for missing trie node +} + +// Unwrap returns the concrete error for missing trie node which +// allows us for further analysis outside. +func (err *MissingNodeError) Unwrap() error { + return err.err } func (err *MissingNodeError) Error() string { - return fmt.Sprintf("missing trie node %x (path %x)", err.NodeHash, err.Path) + if err.Owner == (common.Hash{}) { + return fmt.Sprintf("missing trie node %x (path %x) %v", err.NodeHash, err.Path, err.err) + } + return fmt.Sprintf("missing trie node %x (owner %x) (path %x) %v", err.NodeHash, err.Owner, err.Path, err.err) } diff --git a/trie/hasher.go b/trie/hasher.go index 2949a3ddeece2..183e96c229cad 100644 --- a/trie/hasher.go +++ b/trie/hasher.go @@ -30,7 +30,7 @@ type hasher struct { sha crypto.KeccakState tmp []byte encbuf rlp.EncoderBuffer - parallel bool // Whether to use paralallel threads when hashing + parallel bool // Whether to use parallel threads when hashing } // hasherPool holds pureHashers @@ -191,7 +191,7 @@ func (h *hasher) hashData(data []byte) hashNode { } // proofHash is used to construct trie proofs, and returns the 'collapsed' -// node (for later RLP encoding) aswell as the hashed node -- unless the +// node (for later RLP encoding) as well as the hashed node -- unless the // node is smaller than 32 bytes, in which case it will be returned as is. // This method does not do anything on value- or hash-nodes. func (h *hasher) proofHash(original node) (collapsed, hashed node) { diff --git a/trie/iterator.go b/trie/iterator.go index e0006ee05e3ba..1e76625c6213a 100644 --- a/trie/iterator.go +++ b/trie/iterator.go @@ -375,8 +375,7 @@ func (it *nodeIterator) resolveHash(hash hashNode, path []byte) (node, error) { } } } - resolved, err := it.trie.resolveHash(hash, path) - return resolved, err + return it.trie.resolveHash(hash, path) } func (it *nodeIterator) resolveBlob(hash hashNode, path []byte) ([]byte, error) { diff --git a/trie/iterator_test.go b/trie/iterator_test.go index ea8a46bb43018..e9d822a9a4f27 100644 --- a/trie/iterator_test.go +++ b/trie/iterator_test.go @@ -31,7 +31,7 @@ import ( ) func TestEmptyIterator(t *testing.T) { - trie := newEmpty() + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) iter := trie.NodeIterator(nil) seen := make(map[string]struct{}) @@ -44,7 +44,8 @@ func TestEmptyIterator(t *testing.T) { } func TestIterator(t *testing.T) { - trie := newEmpty() + db := NewDatabase(rawdb.NewMemoryDatabase()) + trie := NewEmpty(db) vals := []struct{ k, v string }{ {"do", "verb"}, {"ether", "wookiedoo"}, @@ -59,8 +60,13 @@ func TestIterator(t *testing.T) { all[val.k] = val.v trie.Update([]byte(val.k), []byte(val.v)) } - trie.Commit(nil) + root, nodes, err := trie.Commit(false) + if err != nil { + t.Fatalf("Failed to commit trie %v", err) + } + db.Update(NewWithNodeSet(nodes)) + trie, _ = New(common.Hash{}, root, db) found := make(map[string]string) it := NewIterator(trie.NodeIterator(nil)) for it.Next() { @@ -80,7 +86,7 @@ type kv struct { } func TestIteratorLargeData(t *testing.T) { - trie := newEmpty() + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) vals := make(map[string]*kv) for i := byte(0); i < 255; i++ { @@ -173,7 +179,7 @@ var testdata2 = []kvs{ } func TestIteratorSeek(t *testing.T) { - trie := newEmpty() + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) for _, val := range testdata1 { trie.Update([]byte(val.k), []byte(val.v)) } @@ -214,17 +220,23 @@ func checkIteratorOrder(want []kvs, it *Iterator) error { } func TestDifferenceIterator(t *testing.T) { - triea := newEmpty() + dba := NewDatabase(rawdb.NewMemoryDatabase()) + triea := NewEmpty(dba) for _, val := range testdata1 { triea.Update([]byte(val.k), []byte(val.v)) } - triea.Commit(nil) + rootA, nodesA, _ := triea.Commit(false) + dba.Update(NewWithNodeSet(nodesA)) + triea, _ = New(common.Hash{}, rootA, dba) - trieb := newEmpty() + dbb := NewDatabase(rawdb.NewMemoryDatabase()) + trieb := NewEmpty(dbb) for _, val := range testdata2 { trieb.Update([]byte(val.k), []byte(val.v)) } - trieb.Commit(nil) + rootB, nodesB, _ := trieb.Commit(false) + dbb.Update(NewWithNodeSet(nodesB)) + trieb, _ = New(common.Hash{}, rootB, dbb) found := make(map[string]string) di, _ := NewDifferenceIterator(triea.NodeIterator(nil), trieb.NodeIterator(nil)) @@ -250,17 +262,23 @@ func TestDifferenceIterator(t *testing.T) { } func TestUnionIterator(t *testing.T) { - triea := newEmpty() + dba := NewDatabase(rawdb.NewMemoryDatabase()) + triea := NewEmpty(dba) for _, val := range testdata1 { triea.Update([]byte(val.k), []byte(val.v)) } - triea.Commit(nil) + rootA, nodesA, _ := triea.Commit(false) + dba.Update(NewWithNodeSet(nodesA)) + triea, _ = New(common.Hash{}, rootA, dba) - trieb := newEmpty() + dbb := NewDatabase(rawdb.NewMemoryDatabase()) + trieb := NewEmpty(dbb) for _, val := range testdata2 { trieb.Update([]byte(val.k), []byte(val.v)) } - trieb.Commit(nil) + rootB, nodesB, _ := trieb.Commit(false) + dbb.Update(NewWithNodeSet(nodesB)) + trieb, _ = New(common.Hash{}, rootB, dbb) di, _ := NewUnionIterator([]NodeIterator{triea.NodeIterator(nil), trieb.NodeIterator(nil)}) it := NewIterator(di) @@ -297,7 +315,7 @@ func TestUnionIterator(t *testing.T) { } func TestIteratorNoDups(t *testing.T) { - tr, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) + tr := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) for _, val := range testdata1 { tr.Update([]byte(val.k), []byte(val.v)) } @@ -312,11 +330,12 @@ func testIteratorContinueAfterError(t *testing.T, memonly bool) { diskdb := memorydb.New() triedb := NewDatabase(diskdb) - tr, _ := New(common.Hash{}, triedb) + tr := NewEmpty(triedb) for _, val := range testdata1 { tr.Update([]byte(val.k), []byte(val.v)) } - tr.Commit(nil) + _, nodes, _ := tr.Commit(false) + triedb.Update(NewWithNodeSet(nodes)) if !memonly { triedb.Commit(tr.Hash(), true, nil) } @@ -337,7 +356,7 @@ func testIteratorContinueAfterError(t *testing.T, memonly bool) { } for i := 0; i < 20; i++ { // Create trie that will load all nodes from DB. - tr, _ := New(tr.Hash(), triedb) + tr, _ := New(common.Hash{}, tr.Hash(), triedb) // Remove a random node from the database. It can't be the root node // because that one is already loaded. @@ -403,11 +422,12 @@ func testIteratorContinueAfterSeekError(t *testing.T, memonly bool) { diskdb := memorydb.New() triedb := NewDatabase(diskdb) - ctr, _ := New(common.Hash{}, triedb) + ctr := NewEmpty(triedb) for _, val := range testdata1 { ctr.Update([]byte(val.k), []byte(val.v)) } - root, _, _ := ctr.Commit(nil) + root, nodes, _ := ctr.Commit(false) + triedb.Update(NewWithNodeSet(nodes)) if !memonly { triedb.Commit(root, true, nil) } @@ -425,7 +445,7 @@ func testIteratorContinueAfterSeekError(t *testing.T, memonly bool) { } // Create a new iterator that seeks to "bars". Seeking can't proceed because // the node is missing. - tr, _ := New(root, triedb) + tr, _ := New(common.Hash{}, root, triedb) it := tr.NodeIterator([]byte("bars")) missing, ok := it.Error().(*MissingNodeError) if !ok { @@ -509,11 +529,11 @@ func (l *loggingDb) Close() error { } // makeLargeTestTrie create a sample test trie -func makeLargeTestTrie() (*Database, *SecureTrie, *loggingDb) { +func makeLargeTestTrie() (*Database, *StateTrie, *loggingDb) { // Create an empty trie logDb := &loggingDb{0, memorydb.New()} triedb := NewDatabase(logDb) - trie, _ := NewSecure(common.Hash{}, triedb) + trie, _ := NewStateTrie(common.Hash{}, common.Hash{}, triedb) // Fill it with some arbitrary data for i := 0; i < 10000; i++ { @@ -525,7 +545,8 @@ func makeLargeTestTrie() (*Database, *SecureTrie, *loggingDb) { val = crypto.Keccak256(val) trie.Update(key, val) } - trie.Commit(nil) + _, nodes, _ := trie.Commit(false) + triedb.Update(NewWithNodeSet(nodes)) // Return the generated trie return triedb, trie, logDb } @@ -546,9 +567,9 @@ func TestNodeIteratorLargeTrie(t *testing.T) { func TestIteratorNodeBlob(t *testing.T) { var ( - db = memorydb.New() - triedb = NewDatabase(db) - trie, _ = New(common.Hash{}, triedb) + db = memorydb.New() + triedb = NewDatabase(db) + trie = NewEmpty(triedb) ) vals := []struct{ k, v string }{ {"do", "verb"}, @@ -564,7 +585,8 @@ func TestIteratorNodeBlob(t *testing.T) { all[val.k] = val.v trie.Update([]byte(val.k), []byte(val.v)) } - trie.Commit(nil) + _, nodes, _ := trie.Commit(false) + triedb.Update(NewWithNodeSet(nodes)) triedb.Cap(0) found := make(map[common.Hash][]byte) diff --git a/trie/node.go b/trie/node.go index bf3f024bb8a78..6ce6551ded8cb 100644 --- a/trie/node.go +++ b/trie/node.go @@ -99,6 +99,7 @@ func (n valueNode) fstring(ind string) string { return fmt.Sprintf("%x ", []byte(n)) } +// mustDecodeNode is a wrapper of decodeNode and panic if any error is encountered. func mustDecodeNode(hash, buf []byte) node { n, err := decodeNode(hash, buf) if err != nil { @@ -107,8 +108,29 @@ func mustDecodeNode(hash, buf []byte) node { return n } -// decodeNode parses the RLP encoding of a trie node. +// mustDecodeNodeUnsafe is a wrapper of decodeNodeUnsafe and panic if any error is +// encountered. +func mustDecodeNodeUnsafe(hash, buf []byte) node { + n, err := decodeNodeUnsafe(hash, buf) + if err != nil { + panic(fmt.Sprintf("node %x: %v", hash, err)) + } + return n +} + +// decodeNode parses the RLP encoding of a trie node. It will deep-copy the passed +// byte slice for decoding, so it's safe to modify the byte slice afterwards. The- +// decode performance of this function is not optimal, but it is suitable for most +// scenarios with low performance requirements and hard to determine whether the +// byte slice be modified or not. func decodeNode(hash, buf []byte) (node, error) { + return decodeNodeUnsafe(hash, common.CopyBytes(buf)) +} + +// decodeNodeUnsafe parses the RLP encoding of a trie node. The passed byte slice +// will be directly referenced by node without bytes deep copy, so the input MUST +// not be changed after. +func decodeNodeUnsafe(hash, buf []byte) (node, error) { if len(buf) == 0 { return nil, io.ErrUnexpectedEOF } @@ -141,7 +163,7 @@ func decodeShort(hash, elems []byte) (node, error) { if err != nil { return nil, fmt.Errorf("invalid value node: %v", err) } - return &shortNode{key, append(valueNode{}, val...), flag}, nil + return &shortNode{key, valueNode(val), flag}, nil } r, _, err := decodeRef(rest) if err != nil { @@ -164,7 +186,7 @@ func decodeFull(hash, elems []byte) (*fullNode, error) { return n, err } if len(val) > 0 { - n.Children[16] = append(valueNode{}, val...) + n.Children[16] = valueNode(val) } return n, nil } @@ -190,7 +212,7 @@ func decodeRef(buf []byte) (node, []byte, error) { // empty node return nil, rest, nil case kind == rlp.String && len(val) == 32: - return append(hashNode{}, val...), rest, nil + return hashNode(val), rest, nil default: return nil, nil, fmt.Errorf("invalid RLP string size %d (want 0 or 32)", len(val)) } diff --git a/trie/node_test.go b/trie/node_test.go index ac1d8fbef3e6b..9b8b33748fa7e 100644 --- a/trie/node_test.go +++ b/trie/node_test.go @@ -20,6 +20,7 @@ import ( "bytes" "testing" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/rlp" ) @@ -92,3 +93,123 @@ func TestDecodeFullNode(t *testing.T) { t.Fatalf("decode full node err: %v", err) } } + +// goos: darwin +// goarch: arm64 +// pkg: github.com/ethereum/go-ethereum/trie +// BenchmarkEncodeShortNode +// BenchmarkEncodeShortNode-8 16878850 70.81 ns/op 48 B/op 1 allocs/op +func BenchmarkEncodeShortNode(b *testing.B) { + node := &shortNode{ + Key: []byte{0x1, 0x2}, + Val: hashNode(randBytes(32)), + } + b.ResetTimer() + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + nodeToBytes(node) + } +} + +// goos: darwin +// goarch: arm64 +// pkg: github.com/ethereum/go-ethereum/trie +// BenchmarkEncodeFullNode +// BenchmarkEncodeFullNode-8 4323273 284.4 ns/op 576 B/op 1 allocs/op +func BenchmarkEncodeFullNode(b *testing.B) { + node := &fullNode{} + for i := 0; i < 16; i++ { + node.Children[i] = hashNode(randBytes(32)) + } + b.ResetTimer() + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + nodeToBytes(node) + } +} + +// goos: darwin +// goarch: arm64 +// pkg: github.com/ethereum/go-ethereum/trie +// BenchmarkDecodeShortNode +// BenchmarkDecodeShortNode-8 7925638 151.0 ns/op 157 B/op 4 allocs/op +func BenchmarkDecodeShortNode(b *testing.B) { + node := &shortNode{ + Key: []byte{0x1, 0x2}, + Val: hashNode(randBytes(32)), + } + blob := nodeToBytes(node) + hash := crypto.Keccak256(blob) + + b.ResetTimer() + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + mustDecodeNode(hash, blob) + } +} + +// goos: darwin +// goarch: arm64 +// pkg: github.com/ethereum/go-ethereum/trie +// BenchmarkDecodeShortNodeUnsafe +// BenchmarkDecodeShortNodeUnsafe-8 9027476 128.6 ns/op 109 B/op 3 allocs/op +func BenchmarkDecodeShortNodeUnsafe(b *testing.B) { + node := &shortNode{ + Key: []byte{0x1, 0x2}, + Val: hashNode(randBytes(32)), + } + blob := nodeToBytes(node) + hash := crypto.Keccak256(blob) + + b.ResetTimer() + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + mustDecodeNodeUnsafe(hash, blob) + } +} + +// goos: darwin +// goarch: arm64 +// pkg: github.com/ethereum/go-ethereum/trie +// BenchmarkDecodeFullNode +// BenchmarkDecodeFullNode-8 1597462 761.9 ns/op 1280 B/op 18 allocs/op +func BenchmarkDecodeFullNode(b *testing.B) { + node := &fullNode{} + for i := 0; i < 16; i++ { + node.Children[i] = hashNode(randBytes(32)) + } + blob := nodeToBytes(node) + hash := crypto.Keccak256(blob) + + b.ResetTimer() + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + mustDecodeNode(hash, blob) + } +} + +// goos: darwin +// goarch: arm64 +// pkg: github.com/ethereum/go-ethereum/trie +// BenchmarkDecodeFullNodeUnsafe +// BenchmarkDecodeFullNodeUnsafe-8 1789070 687.1 ns/op 704 B/op 17 allocs/op +func BenchmarkDecodeFullNodeUnsafe(b *testing.B) { + node := &fullNode{} + for i := 0; i < 16; i++ { + node.Children[i] = hashNode(randBytes(32)) + } + blob := nodeToBytes(node) + hash := crypto.Keccak256(blob) + + b.ResetTimer() + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + mustDecodeNodeUnsafe(hash, blob) + } +} diff --git a/trie/nodeset.go b/trie/nodeset.go new file mode 100644 index 0000000000000..08b9b35ebc877 --- /dev/null +++ b/trie/nodeset.go @@ -0,0 +1,94 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package trie + +import ( + "fmt" + + "github.com/ethereum/go-ethereum/common" +) + +// memoryNode is all the information we know about a single cached trie node +// in the memory. +type memoryNode struct { + hash common.Hash // Node hash, computed by hashing rlp value + size uint16 // Byte size of the useful cached data + node node // Cached collapsed trie node, or raw rlp data +} + +// NodeSet contains all dirty nodes collected during the commit operation. +// Each node is keyed by path. It's not thread-safe to use. +type NodeSet struct { + owner common.Hash // the identifier of the trie + paths []string // the path of dirty nodes, sort by insertion order + nodes map[string]*memoryNode // the map of dirty nodes, keyed by node path + leaves []*leaf // the list of dirty leaves +} + +// NewNodeSet initializes an empty node set to be used for tracking dirty nodes +// from a specific account or storage trie. The owner is zero for the account +// trie and the owning account address hash for storage tries. +func NewNodeSet(owner common.Hash) *NodeSet { + return &NodeSet{ + owner: owner, + nodes: make(map[string]*memoryNode), + } +} + +// add caches node with provided path and node object. +func (set *NodeSet) add(path string, node *memoryNode) { + set.paths = append(set.paths, path) + set.nodes[path] = node +} + +// addLeaf caches the provided leaf node. +func (set *NodeSet) addLeaf(node *leaf) { + set.leaves = append(set.leaves, node) +} + +// Len returns the number of dirty nodes contained in the set. +func (set *NodeSet) Len() int { + return len(set.nodes) +} + +// MergedNodeSet represents a merged dirty node set for a group of tries. +type MergedNodeSet struct { + sets map[common.Hash]*NodeSet +} + +// NewMergedNodeSet initializes an empty merged set. +func NewMergedNodeSet() *MergedNodeSet { + return &MergedNodeSet{sets: make(map[common.Hash]*NodeSet)} +} + +// NewWithNodeSet constructs a merged nodeset with the provided single set. +func NewWithNodeSet(set *NodeSet) *MergedNodeSet { + merged := NewMergedNodeSet() + merged.Merge(set) + return merged +} + +// Merge merges the provided dirty nodes of a trie into the set. The assumption +// is held that no duplicated set belonging to the same trie will be merged twice. +func (set *MergedNodeSet) Merge(other *NodeSet) error { + _, present := set.sets[other.owner] + if present { + return fmt.Errorf("duplicate trie for owner %#x", other.owner) + } + set.sets[other.owner] = other + return nil +} diff --git a/trie/preimages.go b/trie/preimages.go new file mode 100644 index 0000000000000..66f34117c1e83 --- /dev/null +++ b/trie/preimages.go @@ -0,0 +1,95 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package trie + +import ( + "sync" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" +) + +// preimageStore is the store for caching preimages of node key. +type preimageStore struct { + lock sync.RWMutex + disk ethdb.KeyValueStore + preimages map[common.Hash][]byte // Preimages of nodes from the secure trie + preimagesSize common.StorageSize // Storage size of the preimages cache +} + +// newPreimageStore initializes the store for caching preimages. +func newPreimageStore(disk ethdb.KeyValueStore) *preimageStore { + return &preimageStore{ + disk: disk, + preimages: make(map[common.Hash][]byte), + } +} + +// insertPreimage writes a new trie node pre-image to the memory database if it's +// yet unknown. The method will NOT make a copy of the slice, only use if the +// preimage will NOT be changed later on. +func (store *preimageStore) insertPreimage(preimages map[common.Hash][]byte) { + store.lock.Lock() + defer store.lock.Unlock() + + for hash, preimage := range preimages { + if _, ok := store.preimages[hash]; ok { + continue + } + store.preimages[hash] = preimage + store.preimagesSize += common.StorageSize(common.HashLength + len(preimage)) + } +} + +// preimage retrieves a cached trie node pre-image from memory. If it cannot be +// found cached, the method queries the persistent database for the content. +func (store *preimageStore) preimage(hash common.Hash) []byte { + store.lock.RLock() + preimage := store.preimages[hash] + store.lock.RUnlock() + + if preimage != nil { + return preimage + } + return rawdb.ReadPreimage(store.disk, hash) +} + +// commit flushes the cached preimages into the disk. +func (store *preimageStore) commit(force bool) error { + store.lock.Lock() + defer store.lock.Unlock() + + if store.preimagesSize <= 4*1024*1024 && !force { + return nil + } + batch := store.disk.NewBatch() + rawdb.WritePreimages(batch, store.preimages) + if err := batch.Write(); err != nil { + return err + } + store.preimages, store.preimagesSize = make(map[common.Hash][]byte), 0 + return nil +} + +// size returns the current storage size of accumulated preimages. +func (store *preimageStore) size() common.StorageSize { + store.lock.RLock() + defer store.lock.RUnlock() + + return store.preimagesSize +} diff --git a/trie/proof.go b/trie/proof.go index f42dcc761bee7..fa8361eefd58c 100644 --- a/trie/proof.go +++ b/trie/proof.go @@ -22,6 +22,7 @@ import ( "fmt" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" ) @@ -35,9 +36,12 @@ import ( // with the node that proves the absence of the key. func (t *Trie) Prove(key []byte, fromLevel uint, proofDb ethdb.KeyValueWriter) error { // Collect all nodes on the path to key. + var ( + prefix []byte + nodes []node + tn = t.root + ) key = keybytesToHex(key) - var nodes []node - tn := t.root for len(key) > 0 && tn != nil { switch n := tn.(type) { case *shortNode: @@ -46,16 +50,18 @@ func (t *Trie) Prove(key []byte, fromLevel uint, proofDb ethdb.KeyValueWriter) e tn = nil } else { tn = n.Val + prefix = append(prefix, n.Key...) key = key[len(n.Key):] } nodes = append(nodes, n) case *fullNode: tn = n.Children[key[0]] + prefix = append(prefix, key[0]) key = key[1:] nodes = append(nodes, n) case hashNode: var err error - tn, err = t.resolveHash(n, nil) + tn, err = t.resolveHash(n, prefix) if err != nil { log.Error(fmt.Sprintf("Unhandled trie error: %v", err)) return err @@ -94,7 +100,7 @@ func (t *Trie) Prove(key []byte, fromLevel uint, proofDb ethdb.KeyValueWriter) e // If the trie does not contain a value for key, the returned proof contains all // nodes of the longest existing prefix of the key (at least the root node), ending // with the node that proves the absence of the key. -func (t *SecureTrie) Prove(key []byte, fromLevel uint, proofDb ethdb.KeyValueWriter) error { +func (t *StateTrie) Prove(key []byte, fromLevel uint, proofDb ethdb.KeyValueWriter) error { return t.trie.Prove(key, fromLevel, proofDb) } @@ -367,11 +373,12 @@ func unset(parent node, child node, key []byte, pos int, removeLeft bool) error // branch. The parent must be a fullnode. fn := parent.(*fullNode) fn.Children[key[pos-1]] = nil - } else { - // The key of fork shortnode is greater than the - // path(it doesn't belong to the range), keep - // it with the cached hash available. } + //else { + // The key of fork shortnode is greater than the + // path(it doesn't belong to the range), keep + // it with the cached hash available. + //} } else { if bytes.Compare(cld.Key, key[pos:]) > 0 { // The key of fork shortnode is greater than the @@ -379,11 +386,12 @@ func unset(parent node, child node, key []byte, pos int, removeLeft bool) error // branch. The parent must be a fullnode. fn := parent.(*fullNode) fn.Children[key[pos-1]] = nil - } else { - // The key of fork shortnode is less than the - // path(it doesn't belong to the range), keep - // it with the cached hash available. } + //else { + // The key of fork shortnode is less than the + // path(it doesn't belong to the range), keep + // it with the cached hash available. + //} } return nil } @@ -551,7 +559,7 @@ func VerifyRangeProof(rootHash common.Hash, firstKey []byte, lastKey []byte, key } // Rebuild the trie with the leaf stream, the shape of trie // should be same with the original one. - tr := newWithRootNode(root) + tr := &Trie{root: root, db: NewDatabase(rawdb.NewMemoryDatabase())} if empty { tr.root = nil } diff --git a/trie/proof_test.go b/trie/proof_test.go index cdf5cf6050980..61667b20ab132 100644 --- a/trie/proof_test.go +++ b/trie/proof_test.go @@ -80,7 +80,7 @@ func TestProof(t *testing.T) { } func TestOneElementProof(t *testing.T) { - trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) updateString(trie, "k", "v") for i, prover := range makeProvers(trie) { proof := prover([]byte("k")) @@ -131,7 +131,7 @@ func TestBadProof(t *testing.T) { // Tests that missing keys can also be proven. The test explicitly uses a single // entry trie and checks for missing keys both before and after the single entry. func TestMissingKeyProof(t *testing.T) { - trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) updateString(trie, "k", "v") for i, key := range []string{"a", "j", "l", "z"} { @@ -205,7 +205,7 @@ func TestRangeProofWithNonExistentProof(t *testing.T) { proof := memorydb.New() // Short circuit if the decreased key is same with the previous key - first := decreseKey(common.CopyBytes(entries[start].k)) + first := decreaseKey(common.CopyBytes(entries[start].k)) if start != 0 && bytes.Equal(first, entries[start-1].k) { continue } @@ -214,7 +214,7 @@ func TestRangeProofWithNonExistentProof(t *testing.T) { continue } // Short circuit if the increased key is same with the next key - last := increseKey(common.CopyBytes(entries[end-1].k)) + last := increaseKey(common.CopyBytes(entries[end-1].k)) if end != len(entries) && bytes.Equal(last, entries[end].k) { continue } @@ -274,7 +274,7 @@ func TestRangeProofWithInvalidNonExistentProof(t *testing.T) { // Case 1 start, end := 100, 200 - first := decreseKey(common.CopyBytes(entries[start].k)) + first := decreaseKey(common.CopyBytes(entries[start].k)) proof := memorydb.New() if err := trie.Prove(first, 0, proof); err != nil { @@ -297,7 +297,7 @@ func TestRangeProofWithInvalidNonExistentProof(t *testing.T) { // Case 2 start, end = 100, 200 - last := increseKey(common.CopyBytes(entries[end-1].k)) + last := increaseKey(common.CopyBytes(entries[end-1].k)) proof = memorydb.New() if err := trie.Prove(entries[start].k, 0, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) @@ -343,7 +343,7 @@ func TestOneElementRangeProof(t *testing.T) { // One element with left non-existent edge proof start = 1000 - first := decreseKey(common.CopyBytes(entries[start].k)) + first := decreaseKey(common.CopyBytes(entries[start].k)) proof = memorydb.New() if err := trie.Prove(first, 0, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) @@ -358,7 +358,7 @@ func TestOneElementRangeProof(t *testing.T) { // One element with right non-existent edge proof start = 1000 - last := increseKey(common.CopyBytes(entries[start].k)) + last := increaseKey(common.CopyBytes(entries[start].k)) proof = memorydb.New() if err := trie.Prove(entries[start].k, 0, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) @@ -373,7 +373,7 @@ func TestOneElementRangeProof(t *testing.T) { // One element with two non-existent edge proofs start = 1000 - first, last = decreseKey(common.CopyBytes(entries[start].k)), increseKey(common.CopyBytes(entries[start].k)) + first, last = decreaseKey(common.CopyBytes(entries[start].k)), increaseKey(common.CopyBytes(entries[start].k)) proof = memorydb.New() if err := trie.Prove(first, 0, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) @@ -387,7 +387,7 @@ func TestOneElementRangeProof(t *testing.T) { } // Test the mini trie with only a single element. - tinyTrie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) + tinyTrie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) entry := &kv{randBytes(32), randBytes(20), false} tinyTrie.Update(entry.k, entry.v) @@ -459,7 +459,7 @@ func TestAllElementsProof(t *testing.T) { // TestSingleSideRangeProof tests the range starts from zero. func TestSingleSideRangeProof(t *testing.T) { for i := 0; i < 64; i++ { - trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) var entries entrySlice for i := 0; i < 4096; i++ { value := &kv{randBytes(32), randBytes(20), false} @@ -494,7 +494,7 @@ func TestSingleSideRangeProof(t *testing.T) { // TestReverseSingleSideRangeProof tests the range ends with 0xffff...fff. func TestReverseSingleSideRangeProof(t *testing.T) { for i := 0; i < 64; i++ { - trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) var entries entrySlice for i := 0; i < 4096; i++ { value := &kv{randBytes(32), randBytes(20), false} @@ -601,7 +601,7 @@ func TestBadRangeProof(t *testing.T) { // TestGappedRangeProof focuses on the small trie with embedded nodes. // If the gapped node is embedded in the trie, it should be detected too. func TestGappedRangeProof(t *testing.T) { - trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) var entries []*kv // Sorted entries for i := byte(0); i < 10; i++ { value := &kv{common.LeftPadBytes([]byte{i}, 32), []byte{i}, false} @@ -641,9 +641,9 @@ func TestSameSideProofs(t *testing.T) { sort.Sort(entries) pos := 1000 - first := decreseKey(common.CopyBytes(entries[pos].k)) - first = decreseKey(first) - last := decreseKey(common.CopyBytes(entries[pos].k)) + first := decreaseKey(common.CopyBytes(entries[pos].k)) + first = decreaseKey(first) + last := decreaseKey(common.CopyBytes(entries[pos].k)) proof := memorydb.New() if err := trie.Prove(first, 0, proof); err != nil { @@ -657,9 +657,9 @@ func TestSameSideProofs(t *testing.T) { t.Fatalf("Expected error, got nil") } - first = increseKey(common.CopyBytes(entries[pos].k)) - last = increseKey(common.CopyBytes(entries[pos].k)) - last = increseKey(last) + first = increaseKey(common.CopyBytes(entries[pos].k)) + last = increaseKey(common.CopyBytes(entries[pos].k)) + last = increaseKey(last) proof = memorydb.New() if err := trie.Prove(first, 0, proof); err != nil { @@ -675,7 +675,7 @@ func TestSameSideProofs(t *testing.T) { } func TestHasRightElement(t *testing.T) { - trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) var entries entrySlice for i := 0; i < 4096; i++ { value := &kv{randBytes(32), randBytes(20), false} @@ -765,7 +765,7 @@ func TestEmptyRangeProof(t *testing.T) { } for _, c := range cases { proof := memorydb.New() - first := increseKey(common.CopyBytes(entries[c.pos].k)) + first := increaseKey(common.CopyBytes(entries[c.pos].k)) if err := trie.Prove(first, 0, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) } @@ -904,7 +904,7 @@ func mutateByte(b []byte) { } } -func increseKey(key []byte) []byte { +func increaseKey(key []byte) []byte { for i := len(key) - 1; i >= 0; i-- { key[i]++ if key[i] != 0x0 { @@ -914,7 +914,7 @@ func increseKey(key []byte) []byte { return key } -func decreseKey(key []byte) []byte { +func decreaseKey(key []byte) []byte { for i := len(key) - 1; i >= 0; i-- { key[i]-- if key[i] != 0xff { @@ -1028,7 +1028,7 @@ func benchmarkVerifyRangeNoProof(b *testing.B, size int) { } func randomTrie(n int) (*Trie, map[string]*kv) { - trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) vals := make(map[string]*kv) for i := byte(0); i < 100; i++ { value := &kv{common.LeftPadBytes([]byte{i}, 32), []byte{i}, false} @@ -1053,7 +1053,7 @@ func randBytes(n int) []byte { } func nonRandomTrie(n int) (*Trie, map[string]*kv) { - trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) vals := make(map[string]*kv) max := uint64(0xffffffffffffffff) for i := uint64(0); i < uint64(n); i++ { @@ -1078,7 +1078,7 @@ func TestRangeProofKeysWithSharedPrefix(t *testing.T) { common.Hex2Bytes("02"), common.Hex2Bytes("03"), } - trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) for i, key := range keys { trie.Update(key, vals[i]) } diff --git a/trie/secure_trie.go b/trie/secure_trie.go index 248b93544d2fe..3d468f56ee0af 100644 --- a/trie/secure_trie.go +++ b/trie/secure_trie.go @@ -25,24 +25,35 @@ import ( "github.com/ethereum/go-ethereum/rlp" ) -// SecureTrie wraps a trie with key hashing. In a secure trie, all +// SecureTrie is the old name of StateTrie. +// Deprecated: use StateTrie. +type SecureTrie = StateTrie + +// NewSecure creates a new StateTrie. +// Deprecated: use NewStateTrie. +func NewSecure(owner common.Hash, root common.Hash, db *Database) (*SecureTrie, error) { + return NewStateTrie(owner, root, db) +} + +// StateTrie wraps a trie with key hashing. In a secure trie, all // access operations hash the key using keccak256. This prevents // calling code from creating long chains of nodes that // increase the access time. // -// Contrary to a regular trie, a SecureTrie can only be created with +// Contrary to a regular trie, a StateTrie can only be created with // New and must have an attached database. The database also stores // the preimage of each key. // -// SecureTrie is not safe for concurrent use. -type SecureTrie struct { +// StateTrie is not safe for concurrent use. +type StateTrie struct { trie Trie + preimages *preimageStore hashKeyBuf [common.HashLength]byte secKeyCache map[string][]byte - secKeyCacheOwner *SecureTrie // Pointer to self, replace the key cache on mismatch + secKeyCacheOwner *StateTrie // Pointer to self, replace the key cache on mismatch } -// NewSecure creates a trie with an existing root node from a backing database +// NewStateTrie creates a trie with an existing root node from a backing database // and optional intermediate in-memory node pool. // // If root is the zero hash or the sha3 hash of an empty string, the @@ -53,20 +64,20 @@ type SecureTrie struct { // Loaded nodes are kept around until their 'cache generation' expires. // A new cache generation is created by each call to Commit. // cachelimit sets the number of past cache generations to keep. -func NewSecure(root common.Hash, db *Database) (*SecureTrie, error) { +func NewStateTrie(owner common.Hash, root common.Hash, db *Database) (*StateTrie, error) { if db == nil { panic("trie.NewSecure called without a database") } - trie, err := New(root, db) + trie, err := New(owner, root, db) if err != nil { return nil, err } - return &SecureTrie{trie: *trie}, nil + return &StateTrie{trie: *trie, preimages: db.preimages}, nil } // Get returns the value for key stored in the trie. // The value bytes must not be modified by the caller. -func (t *SecureTrie) Get(key []byte) []byte { +func (t *StateTrie) Get(key []byte) []byte { res, err := t.TryGet(key) if err != nil { log.Error(fmt.Sprintf("Unhandled trie error: %v", err)) @@ -77,19 +88,50 @@ func (t *SecureTrie) Get(key []byte) []byte { // TryGet returns the value for key stored in the trie. // The value bytes must not be modified by the caller. // If a node was not found in the database, a MissingNodeError is returned. -func (t *SecureTrie) TryGet(key []byte) ([]byte, error) { +func (t *StateTrie) TryGet(key []byte) ([]byte, error) { return t.trie.TryGet(t.hashKey(key)) } +func (t *StateTrie) TryGetAccount(key []byte) (*types.StateAccount, error) { + var ret types.StateAccount + res, err := t.trie.TryGet(t.hashKey(key)) + if err != nil { + log.Error(fmt.Sprintf("Unhandled trie error: %v", err)) + return &ret, err + } + if res == nil { + return nil, nil + } + err = rlp.DecodeBytes(res, &ret) + return &ret, err +} + +// TryGetAccountWithPreHashedKey does the same thing as TryGetAccount, however +// it expects a key that is already hashed. This constitutes an abstraction leak, +// since the client code needs to know the key format. +func (t *StateTrie) TryGetAccountWithPreHashedKey(key []byte) (*types.StateAccount, error) { + var ret types.StateAccount + res, err := t.trie.TryGet(key) + if err != nil { + log.Error(fmt.Sprintf("Unhandled trie error: %v", err)) + return &ret, err + } + if res == nil { + return nil, nil + } + err = rlp.DecodeBytes(res, &ret) + return &ret, err +} + // TryGetNode attempts to retrieve a trie node by compact-encoded path. It is not // possible to use keybyte-encoding as the path might contain odd nibbles. -func (t *SecureTrie) TryGetNode(path []byte) ([]byte, int, error) { +func (t *StateTrie) TryGetNode(path []byte) ([]byte, int, error) { return t.trie.TryGetNode(path) } // TryUpdateAccount account will abstract the write of an account to the // secure trie. -func (t *SecureTrie) TryUpdateAccount(key []byte, acc *types.StateAccount) error { +func (t *StateTrie) TryUpdateAccount(key []byte, acc *types.StateAccount) error { hk := t.hashKey(key) data, err := rlp.EncodeToBytes(acc) if err != nil { @@ -108,7 +150,7 @@ func (t *SecureTrie) TryUpdateAccount(key []byte, acc *types.StateAccount) error // // The value bytes must not be modified by the caller while they are // stored in the trie. -func (t *SecureTrie) Update(key, value []byte) { +func (t *StateTrie) Update(key, value []byte) { if err := t.TryUpdate(key, value); err != nil { log.Error(fmt.Sprintf("Unhandled trie error: %v", err)) } @@ -122,7 +164,7 @@ func (t *SecureTrie) Update(key, value []byte) { // stored in the trie. // // If a node was not found in the database, a MissingNodeError is returned. -func (t *SecureTrie) TryUpdate(key, value []byte) error { +func (t *StateTrie) TryUpdate(key, value []byte) error { hk := t.hashKey(key) err := t.trie.TryUpdate(hk, value) if err != nil { @@ -133,7 +175,7 @@ func (t *SecureTrie) TryUpdate(key, value []byte) error { } // Delete removes any existing value for key from the trie. -func (t *SecureTrie) Delete(key []byte) { +func (t *StateTrie) Delete(key []byte) { if err := t.TryDelete(key); err != nil { log.Error(fmt.Sprintf("Unhandled trie error: %v", err)) } @@ -141,7 +183,14 @@ func (t *SecureTrie) Delete(key []byte) { // TryDelete removes any existing value for key from the trie. // If a node was not found in the database, a MissingNodeError is returned. -func (t *SecureTrie) TryDelete(key []byte) error { +func (t *StateTrie) TryDelete(key []byte) error { + hk := t.hashKey(key) + delete(t.getSecKeyCache(), string(hk)) + return t.trie.TryDelete(hk) +} + +// TryDeleteACcount abstracts an account deletion from the trie. +func (t *StateTrie) TryDeleteAccount(key []byte) error { hk := t.hashKey(key) delete(t.getSecKeyCache(), string(hk)) return t.trie.TryDelete(hk) @@ -149,58 +198,64 @@ func (t *SecureTrie) TryDelete(key []byte) error { // GetKey returns the sha3 preimage of a hashed key that was // previously used to store a value. -func (t *SecureTrie) GetKey(shaKey []byte) []byte { +func (t *StateTrie) GetKey(shaKey []byte) []byte { if key, ok := t.getSecKeyCache()[string(shaKey)]; ok { return key } - return t.trie.db.preimage(common.BytesToHash(shaKey)) + if t.preimages == nil { + return nil + } + return t.preimages.preimage(common.BytesToHash(shaKey)) } -// Commit writes all nodes and the secure hash pre-images to the trie's database. -// Nodes are stored with their sha3 hash as the key. -// -// Committing flushes nodes from memory. Subsequent Get calls will load nodes -// from the database. -func (t *SecureTrie) Commit(onleaf LeafCallback) (common.Hash, int, error) { +// Commit collects all dirty nodes in the trie and replace them with the +// corresponding node hash. All collected nodes(including dirty leaves if +// collectLeaf is true) will be encapsulated into a nodeset for return. +// The returned nodeset can be nil if the trie is clean(nothing to commit). +// All cached preimages will be also flushed if preimages recording is enabled. +// Once the trie is committed, it's not usable anymore. A new trie must +// be created with new root and updated trie database for following usage +func (t *StateTrie) Commit(collectLeaf bool) (common.Hash, *NodeSet, error) { // Write all the pre-images to the actual disk database if len(t.getSecKeyCache()) > 0 { - if t.trie.db.preimages != nil { // Ugly direct check but avoids the below write lock - t.trie.db.lock.Lock() + if t.preimages != nil { + preimages := make(map[common.Hash][]byte) for hk, key := range t.secKeyCache { - t.trie.db.insertPreimage(common.BytesToHash([]byte(hk)), key) + preimages[common.BytesToHash([]byte(hk))] = key } - t.trie.db.lock.Unlock() + t.preimages.insertPreimage(preimages) } t.secKeyCache = make(map[string][]byte) } // Commit the trie to its intermediate node database - return t.trie.Commit(onleaf) + return t.trie.Commit(collectLeaf) } -// Hash returns the root hash of SecureTrie. It does not write to the +// Hash returns the root hash of StateTrie. It does not write to the // database and can be used even if the trie doesn't have one. -func (t *SecureTrie) Hash() common.Hash { +func (t *StateTrie) Hash() common.Hash { return t.trie.Hash() } -// Copy returns a copy of SecureTrie. -func (t *SecureTrie) Copy() *SecureTrie { - return &SecureTrie{ +// Copy returns a copy of StateTrie. +func (t *StateTrie) Copy() *StateTrie { + return &StateTrie{ trie: *t.trie.Copy(), + preimages: t.preimages, secKeyCache: t.secKeyCache, } } // NodeIterator returns an iterator that returns nodes of the underlying trie. Iteration // starts at the key after the given start key. -func (t *SecureTrie) NodeIterator(start []byte) NodeIterator { +func (t *StateTrie) NodeIterator(start []byte) NodeIterator { return t.trie.NodeIterator(start) } // hashKey returns the hash of key as an ephemeral buffer. // The caller must not hold onto the return value because it will become // invalid on the next call to hashKey or secKey. -func (t *SecureTrie) hashKey(key []byte) []byte { +func (t *StateTrie) hashKey(key []byte) []byte { h := newHasher(false) h.sha.Reset() h.sha.Write(key) @@ -212,7 +267,7 @@ func (t *SecureTrie) hashKey(key []byte) []byte { // getSecKeyCache returns the current secure key cache, creating a new one if // ownership changed (i.e. the current secure trie is a copy of another owning // the actual cache). -func (t *SecureTrie) getSecKeyCache() map[string][]byte { +func (t *StateTrie) getSecKeyCache() map[string][]byte { if t != t.secKeyCacheOwner { t.secKeyCacheOwner = t t.secKeyCache = make(map[string][]byte) diff --git a/trie/secure_trie_test.go b/trie/secure_trie_test.go index a3ece84b57120..862c3a3ec43df 100644 --- a/trie/secure_trie_test.go +++ b/trie/secure_trie_test.go @@ -18,6 +18,7 @@ package trie import ( "bytes" + "fmt" "runtime" "sync" "testing" @@ -27,16 +28,16 @@ import ( "github.com/ethereum/go-ethereum/ethdb/memorydb" ) -func newEmptySecure() *SecureTrie { - trie, _ := NewSecure(common.Hash{}, NewDatabase(memorydb.New())) +func newEmptySecure() *StateTrie { + trie, _ := NewStateTrie(common.Hash{}, common.Hash{}, NewDatabase(memorydb.New())) return trie } -// makeTestSecureTrie creates a large enough secure trie for testing. -func makeTestSecureTrie() (*Database, *SecureTrie, map[string][]byte) { +// makeTestStateTrie creates a large enough secure trie for testing. +func makeTestStateTrie() (*Database, *StateTrie, map[string][]byte) { // Create an empty trie triedb := NewDatabase(memorydb.New()) - trie, _ := NewSecure(common.Hash{}, triedb) + trie, _ := NewStateTrie(common.Hash{}, common.Hash{}, triedb) // Fill it with some arbitrary data content := make(map[string][]byte) @@ -57,9 +58,15 @@ func makeTestSecureTrie() (*Database, *SecureTrie, map[string][]byte) { trie.Update(key, val) } } - trie.Commit(nil) - - // Return the generated trie + root, nodes, err := trie.Commit(false) + if err != nil { + panic(fmt.Errorf("failed to commit trie %v", err)) + } + if err := triedb.Update(NewWithNodeSet(nodes)); err != nil { + panic(fmt.Errorf("failed to commit db %v", err)) + } + // Re-create the trie based on the new state + trie, _ = NewSecure(common.Hash{}, root, triedb) return triedb, trie, content } @@ -105,16 +112,16 @@ func TestSecureGetKey(t *testing.T) { } } -func TestSecureTrieConcurrency(t *testing.T) { +func TestStateTrieConcurrency(t *testing.T) { // Create an initial trie and copy if for concurrent access - _, trie, _ := makeTestSecureTrie() + _, trie, _ := makeTestStateTrie() threads := runtime.NumCPU() - tries := make([]*SecureTrie, threads) + tries := make([]*StateTrie, threads) for i := 0; i < threads; i++ { tries[i] = trie.Copy() } - // Start a batch of goroutines interactng with the trie + // Start a batch of goroutines interacting with the trie pend := new(sync.WaitGroup) pend.Add(threads) for i := 0; i < threads; i++ { @@ -135,7 +142,7 @@ func TestSecureTrieConcurrency(t *testing.T) { tries[index].Update(key, val) } } - tries[index].Commit(nil) + tries[index].Commit(false) }(i) } // Wait for all threads to finish diff --git a/trie/stacktrie.go b/trie/stacktrie.go index b38bb01b0fb32..a22fa0dd6797f 100644 --- a/trie/stacktrie.go +++ b/trie/stacktrie.go @@ -38,9 +38,10 @@ var stPool = sync.Pool{ }, } -func stackTrieFromPool(db ethdb.KeyValueWriter) *StackTrie { +func stackTrieFromPool(db ethdb.KeyValueWriter, owner common.Hash) *StackTrie { st := stPool.Get().(*StackTrie) st.db = db + st.owner = owner return st } @@ -53,6 +54,7 @@ func returnToPool(st *StackTrie) { // in order. Once it determines that a subtree will no longer be inserted // into, it will hash it and free up the memory it uses. type StackTrie struct { + owner common.Hash // the owner of the trie nodeType uint8 // node type (as in branch, ext, leaf) val []byte // value contained by this node if it's a leaf key []byte // key chunk covered by this (leaf|ext) node @@ -68,6 +70,16 @@ func NewStackTrie(db ethdb.KeyValueWriter) *StackTrie { } } +// NewStackTrieWithOwner allocates and initializes an empty trie, but with +// the additional owner field. +func NewStackTrieWithOwner(db ethdb.KeyValueWriter, owner common.Hash) *StackTrie { + return &StackTrie{ + owner: owner, + nodeType: emptyNode, + db: db, + } +} + // NewFromBinary initialises a serialized stacktrie with the given db. func NewFromBinary(data []byte, db ethdb.KeyValueWriter) (*StackTrie, error) { var st StackTrie @@ -88,10 +100,12 @@ func (st *StackTrie) MarshalBinary() (data []byte, err error) { w = bufio.NewWriter(&b) ) if err := gob.NewEncoder(w).Encode(struct { - Nodetype uint8 + Owner common.Hash + NodeType uint8 Val []byte Key []byte }{ + st.owner, st.nodeType, st.val, st.key, @@ -122,12 +136,14 @@ func (st *StackTrie) UnmarshalBinary(data []byte) error { func (st *StackTrie) unmarshalBinary(r io.Reader) error { var dec struct { - Nodetype uint8 + Owner common.Hash + NodeType uint8 Val []byte Key []byte } gob.NewDecoder(r).Decode(&dec) - st.nodeType = dec.Nodetype + st.owner = dec.Owner + st.nodeType = dec.NodeType st.val = dec.Val st.key = dec.Key @@ -154,16 +170,16 @@ func (st *StackTrie) setDb(db ethdb.KeyValueWriter) { } } -func newLeaf(key, val []byte, db ethdb.KeyValueWriter) *StackTrie { - st := stackTrieFromPool(db) +func newLeaf(owner common.Hash, key, val []byte, db ethdb.KeyValueWriter) *StackTrie { + st := stackTrieFromPool(db, owner) st.nodeType = leafNode st.key = append(st.key, key...) st.val = val return st } -func newExt(key []byte, child *StackTrie, db ethdb.KeyValueWriter) *StackTrie { - st := stackTrieFromPool(db) +func newExt(owner common.Hash, key []byte, child *StackTrie, db ethdb.KeyValueWriter) *StackTrie { + st := stackTrieFromPool(db, owner) st.nodeType = extNode st.key = append(st.key, key...) st.children[0] = child @@ -196,6 +212,7 @@ func (st *StackTrie) Update(key, value []byte) { } func (st *StackTrie) Reset() { + st.owner = common.Hash{} st.db = nil st.key = st.key[:0] st.val = nil @@ -236,7 +253,7 @@ func (st *StackTrie) insert(key, value []byte) { // Add new child if st.children[idx] == nil { - st.children[idx] = newLeaf(key[1:], value, st.db) + st.children[idx] = newLeaf(st.owner, key[1:], value, st.db) } else { st.children[idx].insert(key[1:], value) } @@ -262,7 +279,7 @@ func (st *StackTrie) insert(key, value []byte) { // node directly. var n *StackTrie if diffidx < len(st.key)-1 { - n = newExt(st.key[diffidx+1:], st.children[0], st.db) + n = newExt(st.owner, st.key[diffidx+1:], st.children[0], st.db) } else { // Break on the last byte, no need to insert // an extension node: reuse the current node @@ -282,12 +299,12 @@ func (st *StackTrie) insert(key, value []byte) { // the common prefix is at least one byte // long, insert a new intermediate branch // node. - st.children[0] = stackTrieFromPool(st.db) + st.children[0] = stackTrieFromPool(st.db, st.owner) st.children[0].nodeType = branchNode p = st.children[0] } // Create a leaf for the inserted part - o := newLeaf(key[diffidx+1:], value, st.db) + o := newLeaf(st.owner, key[diffidx+1:], value, st.db) // Insert both child leaves where they belong: origIdx := st.key[diffidx] @@ -323,7 +340,7 @@ func (st *StackTrie) insert(key, value []byte) { // Convert current node into an ext, // and insert a child branch node. st.nodeType = extNode - st.children[0] = NewStackTrie(st.db) + st.children[0] = NewStackTrieWithOwner(st.db, st.owner) st.children[0].nodeType = branchNode p = st.children[0] } @@ -332,11 +349,11 @@ func (st *StackTrie) insert(key, value []byte) { // value and another containing the new value. The child leaf // is hashed directly in order to free up some memory. origIdx := st.key[diffidx] - p.children[origIdx] = newLeaf(st.key[diffidx+1:], st.val, st.db) + p.children[origIdx] = newLeaf(st.owner, st.key[diffidx+1:], st.val, st.db) p.children[origIdx].hash() newIdx := key[diffidx] - p.children[newIdx] = newLeaf(key[diffidx+1:], value, st.db) + p.children[newIdx] = newLeaf(st.owner, key[diffidx+1:], value, st.db) // Finally, cut off the key part that has been passed // over to the children. diff --git a/trie/stacktrie_test.go b/trie/stacktrie_test.go index e57df60369bf2..069e4981d71ab 100644 --- a/trie/stacktrie_test.go +++ b/trie/stacktrie_test.go @@ -188,7 +188,7 @@ func TestStackTrieInsertAndHash(t *testing.T) { func TestSizeBug(t *testing.T) { st := NewStackTrie(nil) - nt, _ := New(common.Hash{}, NewDatabase(memorydb.New())) + nt := NewEmpty(NewDatabase(memorydb.New())) leaf := common.FromHex("290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563") value := common.FromHex("94cf40d0d2b44f2b66e07cace1372ca42b73cf21a3") @@ -203,7 +203,7 @@ func TestSizeBug(t *testing.T) { func TestEmptyBug(t *testing.T) { st := NewStackTrie(nil) - nt, _ := New(common.Hash{}, NewDatabase(memorydb.New())) + nt := NewEmpty(NewDatabase(memorydb.New())) //leaf := common.FromHex("290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563") //value := common.FromHex("94cf40d0d2b44f2b66e07cace1372ca42b73cf21a3") @@ -229,7 +229,7 @@ func TestEmptyBug(t *testing.T) { func TestValLength56(t *testing.T) { st := NewStackTrie(nil) - nt, _ := New(common.Hash{}, NewDatabase(memorydb.New())) + nt := NewEmpty(NewDatabase(memorydb.New())) //leaf := common.FromHex("290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563") //value := common.FromHex("94cf40d0d2b44f2b66e07cace1372ca42b73cf21a3") @@ -254,7 +254,8 @@ func TestValLength56(t *testing.T) { // which causes a lot of node-within-node. This case was found via fuzzing. func TestUpdateSmallNodes(t *testing.T) { st := NewStackTrie(nil) - nt, _ := New(common.Hash{}, NewDatabase(memorydb.New())) + nt := NewEmpty(NewDatabase(memorydb.New())) + kvs := []struct { K string V string @@ -282,7 +283,8 @@ func TestUpdateSmallNodes(t *testing.T) { func TestUpdateVariableKeys(t *testing.T) { t.SkipNow() st := NewStackTrie(nil) - nt, _ := New(common.Hash{}, NewDatabase(memorydb.New())) + nt := NewEmpty(NewDatabase(memorydb.New())) + kvs := []struct { K string V string @@ -343,7 +345,6 @@ func TestStacktrieNotModifyValues(t *testing.T) { if !bytes.Equal(have, want) { t.Fatalf("item %d, have %#x want %#x", i, have, want) } - } } @@ -352,7 +353,7 @@ func TestStacktrieNotModifyValues(t *testing.T) { func TestStacktrieSerialization(t *testing.T) { var ( st = NewStackTrie(nil) - nt, _ = New(common.Hash{}, NewDatabase(memorydb.New())) + nt = NewEmpty(NewDatabase(memorydb.New())) keyB = big.NewInt(1) keyDelta = big.NewInt(1) vals [][]byte diff --git a/trie/sync.go b/trie/sync.go index db51dd4b036af..862ce7e16e6c4 100644 --- a/trie/sync.go +++ b/trie/sync.go @@ -19,11 +19,13 @@ package trie import ( "errors" "fmt" + "sync" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/prque" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/log" ) // ErrNotRequested is returned by the trie sync when it's requested to process a @@ -39,19 +41,6 @@ var ErrAlreadyProcessed = errors.New("already processed") // memory if the node was configured with a significant number of peers. const maxFetchesPerDepth = 16384 -// request represents a scheduled or already in-flight state retrieval request. -type request struct { - path []byte // Merkle path leading to this node for prioritization - hash common.Hash // Hash of the node data content to retrieve - data []byte // Data content of the node, cached until all subtrees complete - code bool // Whether this is a code entry - - parents []*request // Parent state nodes referencing this entry (notify all upon completion) - deps int // Number of dependencies before allowed to commit this node - - callback LeafCallback // Callback to invoke if a leaf node it reached on this branch -} - // SyncPath is a path tuple identifying a particular trie node either in a single // trie (account) or a layered trie (account -> storage). // @@ -85,30 +74,57 @@ func NewSyncPath(path []byte) SyncPath { return SyncPath{hexToKeybytes(path[:64]), hexToCompact(path[64:])} } -// SyncResult is a response with requested data along with it's hash. -type SyncResult struct { - Hash common.Hash // Hash of the originally unknown trie node - Data []byte // Data content of the retrieved node +// nodeRequest represents a scheduled or already in-flight trie node retrieval request. +type nodeRequest struct { + hash common.Hash // Hash of the trie node to retrieve + path []byte // Merkle path leading to this node for prioritization + data []byte // Data content of the node, cached until all subtrees complete + + parent *nodeRequest // Parent state node referencing this entry + deps int // Number of dependencies before allowed to commit this node + callback LeafCallback // Callback to invoke if a leaf node it reached on this branch +} + +// codeRequest represents a scheduled or already in-flight bytecode retrieval request. +type codeRequest struct { + hash common.Hash // Hash of the contract bytecode to retrieve + path []byte // Merkle path leading to this node for prioritization + data []byte // Data content of the node, cached until all subtrees complete + parents []*nodeRequest // Parent state nodes referencing this entry (notify all upon completion) +} + +// NodeSyncResult is a response with requested trie node along with its node path. +type NodeSyncResult struct { + Path string // Path of the originally unknown trie node + Data []byte // Data content of the retrieved trie node +} + +// CodeSyncResult is a response with requested bytecode along with its hash. +type CodeSyncResult struct { + Hash common.Hash // Hash the originally unknown bytecode + Data []byte // Data content of the retrieved bytecode } // syncMemBatch is an in-memory buffer of successfully downloaded but not yet // persisted data items. type syncMemBatch struct { - nodes map[common.Hash][]byte // In-memory membatch of recently completed nodes - codes map[common.Hash][]byte // In-memory membatch of recently completed codes + nodes map[string][]byte // In-memory membatch of recently completed nodes + hashes map[string]common.Hash // Hashes of recently completed nodes + codes map[common.Hash][]byte // In-memory membatch of recently completed codes } // newSyncMemBatch allocates a new memory-buffer for not-yet persisted trie nodes. func newSyncMemBatch() *syncMemBatch { return &syncMemBatch{ - nodes: make(map[common.Hash][]byte), - codes: make(map[common.Hash][]byte), + nodes: make(map[string][]byte), + hashes: make(map[string]common.Hash), + codes: make(map[common.Hash][]byte), } } -// hasNode reports the trie node with specific hash is already cached. -func (batch *syncMemBatch) hasNode(hash common.Hash) bool { - _, ok := batch.nodes[hash] +// hasNode reports the trie node with specific path is already cached. +func (batch *syncMemBatch) hasNode(path []byte) bool { + _, ok := batch.nodes[string(path)] return ok } @@ -122,12 +138,12 @@ func (batch *syncMemBatch) hasCode(hash common.Hash) bool { // unknown trie hashes to retrieve, accepts node data associated with said hashes // and reconstructs the trie step by step until all is done. type Sync struct { - database ethdb.KeyValueReader // Persistent database to check for existing entries - membatch *syncMemBatch // Memory buffer to avoid frequent database writes - nodeReqs map[common.Hash]*request // Pending requests pertaining to a trie node hash - codeReqs map[common.Hash]*request // Pending requests pertaining to a code hash - queue *prque.Prque // Priority queue with the pending requests - fetches map[int]int // Number of active fetches per trie node depth + database ethdb.KeyValueReader // Persistent database to check for existing entries + membatch *syncMemBatch // Memory buffer to avoid frequent database writes + nodeReqs map[string]*nodeRequest // Pending requests pertaining to a trie node path + codeReqs map[common.Hash]*codeRequest // Pending requests pertaining to a code hash + queue *prque.Prque // Priority queue with the pending requests + fetches map[int]int // Number of active fetches per trie node depth } // NewSync creates a new trie data download scheduler. @@ -135,51 +151,51 @@ func NewSync(root common.Hash, database ethdb.KeyValueReader, callback LeafCallb ts := &Sync{ database: database, membatch: newSyncMemBatch(), - nodeReqs: make(map[common.Hash]*request), - codeReqs: make(map[common.Hash]*request), + nodeReqs: make(map[string]*nodeRequest), + codeReqs: make(map[common.Hash]*codeRequest), queue: prque.New(nil), fetches: make(map[int]int), } - ts.AddSubTrie(root, nil, common.Hash{}, callback) + ts.AddSubTrie(root, nil, common.Hash{}, nil, callback) return ts } -// AddSubTrie registers a new trie to the sync code, rooted at the designated parent. -func (s *Sync) AddSubTrie(root common.Hash, path []byte, parent common.Hash, callback LeafCallback) { +// AddSubTrie registers a new trie to the sync code, rooted at the designated +// parent for completion tracking. The given path is a unique node path in +// hex format and contain all the parent path if it's layered trie node. +func (s *Sync) AddSubTrie(root common.Hash, path []byte, parent common.Hash, parentPath []byte, callback LeafCallback) { // Short circuit if the trie is empty or already known if root == emptyRoot { return } - if s.membatch.hasNode(root) { + if s.membatch.hasNode(path) { return } - // If database says this is a duplicate, then at least the trie node is - // present, and we hold the assumption that it's NOT legacy contract code. if rawdb.HasTrieNode(s.database, root) { return } // Assemble the new sub-trie sync request - req := &request{ - path: path, + req := &nodeRequest{ hash: root, + path: path, callback: callback, } // If this sub-trie has a designated parent, link them together if parent != (common.Hash{}) { - ancestor := s.nodeReqs[parent] + ancestor := s.nodeReqs[string(parentPath)] if ancestor == nil { panic(fmt.Sprintf("sub-trie ancestor not found: %x", parent)) } ancestor.deps++ - req.parents = append(req.parents, ancestor) + req.parent = ancestor } - s.schedule(req) + s.scheduleNodeRequest(req) } // AddCodeEntry schedules the direct retrieval of a contract code that should not // be interpreted as a trie node, but rather accepted and stored into the database // as is. -func (s *Sync) AddCodeEntry(hash common.Hash, path []byte, parent common.Hash) { +func (s *Sync) AddCodeEntry(hash common.Hash, path []byte, parent common.Hash, parentPath []byte) { // Short circuit if the entry is empty or already known if hash == emptyState { return @@ -196,30 +212,29 @@ func (s *Sync) AddCodeEntry(hash common.Hash, path []byte, parent common.Hash) { return } // Assemble the new sub-trie sync request - req := &request{ + req := &codeRequest{ path: path, hash: hash, - code: true, } // If this sub-trie has a designated parent, link them together if parent != (common.Hash{}) { - ancestor := s.nodeReqs[parent] // the parent of codereq can ONLY be nodereq + ancestor := s.nodeReqs[string(parentPath)] // the parent of codereq can ONLY be nodereq if ancestor == nil { panic(fmt.Sprintf("raw-entry ancestor not found: %x", parent)) } ancestor.deps++ req.parents = append(req.parents, ancestor) } - s.schedule(req) + s.scheduleCodeRequest(req) } // Missing retrieves the known missing nodes from the trie for retrieval. To aid // both eth/6x style fast sync and snap/1x style state sync, the paths of trie // nodes are returned too, as well as separate hash list for codes. -func (s *Sync) Missing(max int) (nodes []common.Hash, paths []SyncPath, codes []common.Hash) { +func (s *Sync) Missing(max int) ([]string, []common.Hash, []common.Hash) { var ( + nodePaths []string nodeHashes []common.Hash - nodePaths []SyncPath codeHashes []common.Hash ) for !s.queue.Empty() && (max == 0 || len(nodeHashes)+len(codeHashes) < max) { @@ -235,62 +250,76 @@ func (s *Sync) Missing(max int) (nodes []common.Hash, paths []SyncPath, codes [] s.queue.Pop() s.fetches[depth]++ - hash := item.(common.Hash) - if req, ok := s.nodeReqs[hash]; ok { - nodeHashes = append(nodeHashes, hash) - nodePaths = append(nodePaths, NewSyncPath(req.path)) - } else { - codeHashes = append(codeHashes, hash) + switch item := item.(type) { + case common.Hash: + codeHashes = append(codeHashes, item) + case string: + req, ok := s.nodeReqs[item] + if !ok { + log.Error("Missing node request", "path", item) + continue // System very wrong, shouldn't happen + } + nodePaths = append(nodePaths, item) + nodeHashes = append(nodeHashes, req.hash) } } - return nodeHashes, nodePaths, codeHashes + return nodePaths, nodeHashes, codeHashes } -// Process injects the received data for requested item. Note it can +// ProcessCode injects the received data for requested item. Note it can // happpen that the single response commits two pending requests(e.g. // there are two requests one for code and one for node but the hash // is same). In this case the second response for the same hash will // be treated as "non-requested" item or "already-processed" item but // there is no downside. -func (s *Sync) Process(result SyncResult) error { - // If the item was not requested either for code or node, bail out - if s.nodeReqs[result.Hash] == nil && s.codeReqs[result.Hash] == nil { +func (s *Sync) ProcessCode(result CodeSyncResult) error { + // If the code was not requested or it's already processed, bail out + req := s.codeReqs[result.Hash] + if req == nil { return ErrNotRequested } - // There is an pending code request for this data, commit directly - var filled bool - if req := s.codeReqs[result.Hash]; req != nil && req.data == nil { - filled = true - req.data = result.Data - s.commit(req) - } - // There is an pending node request for this data, fill it. - if req := s.nodeReqs[result.Hash]; req != nil && req.data == nil { - filled = true - // Decode the node data content and update the request - node, err := decodeNode(result.Hash[:], result.Data) - if err != nil { - return err - } - req.data = result.Data + if req.data != nil { + return ErrAlreadyProcessed + } + req.data = result.Data + return s.commitCodeRequest(req) +} - // Create and schedule a request for all the children nodes - requests, err := s.children(req, node) - if err != nil { - return err - } - if len(requests) == 0 && req.deps == 0 { - s.commit(req) - } else { - req.deps += len(requests) - for _, child := range requests { - s.schedule(child) - } - } +// ProcessNode injects the received data for requested item. Note it can +// happen that the single response commits two pending requests(e.g. +// there are two requests one for code and one for node but the hash +// is same). In this case the second response for the same hash will +// be treated as "non-requested" item or "already-processed" item but +// there is no downside. +func (s *Sync) ProcessNode(result NodeSyncResult) error { + // If the trie node was not requested or it's already processed, bail out + req := s.nodeReqs[result.Path] + if req == nil { + return ErrNotRequested } - if !filled { + if req.data != nil { return ErrAlreadyProcessed } + // Decode the node data content and update the request + node, err := decodeNode(req.hash.Bytes(), result.Data) + if err != nil { + return err + } + req.data = result.Data + + // Create and schedule a request for all the children nodes + requests, err := s.children(req, node) + if err != nil { + return err + } + if len(requests) == 0 && req.deps == 0 { + s.commitNodeRequest(req) + } else { + req.deps += len(requests) + for _, child := range requests { + s.scheduleNodeRequest(child) + } + } return nil } @@ -298,11 +327,11 @@ func (s *Sync) Process(result SyncResult) error { // storage, returning any occurred error. func (s *Sync) Commit(dbw ethdb.Batch) error { // Dump the membatch into a database dbw - for key, value := range s.membatch.nodes { - rawdb.WriteTrieNode(dbw, key, value) + for path, value := range s.membatch.nodes { + rawdb.WriteTrieNode(dbw, s.membatch.hashes[path], value) } - for key, value := range s.membatch.codes { - rawdb.WriteCode(dbw, key, value) + for hash, value := range s.membatch.codes { + rawdb.WriteCode(dbw, hash, value) } // Drop the membatch data and return s.membatch = newSyncMemBatch() @@ -317,23 +346,31 @@ func (s *Sync) Pending() int { // schedule inserts a new state retrieval request into the fetch queue. If there // is already a pending request for this node, the new request will be discarded // and only a parent reference added to the old one. -func (s *Sync) schedule(req *request) { - var reqset = s.nodeReqs - if req.code { - reqset = s.codeReqs +func (s *Sync) scheduleNodeRequest(req *nodeRequest) { + s.nodeReqs[string(req.path)] = req + + // Schedule the request for future retrieval. This queue is shared + // by both node requests and code requests. + prio := int64(len(req.path)) << 56 // depth >= 128 will never happen, storage leaves will be included in their parents + for i := 0; i < 14 && i < len(req.path); i++ { + prio |= int64(15-req.path[i]) << (52 - i*4) // 15-nibble => lexicographic order } + s.queue.Push(string(req.path), prio) +} + +// schedule inserts a new state retrieval request into the fetch queue. If there +// is already a pending request for this node, the new request will be discarded +// and only a parent reference added to the old one. +func (s *Sync) scheduleCodeRequest(req *codeRequest) { // If we're already requesting this node, add a new reference and stop - if old, ok := reqset[req.hash]; ok { + if old, ok := s.codeReqs[req.hash]; ok { old.parents = append(old.parents, req.parents...) return } - reqset[req.hash] = req + s.codeReqs[req.hash] = req // Schedule the request for future retrieval. This queue is shared - // by both node requests and code requests. It can happen that there - // is a trie node and code has same hash. In this case two elements - // with same hash and same or different depth will be pushed. But it's - // ok the worst case is the second response will be treated as duplicated. + // by both node requests and code requests. prio := int64(len(req.path)) << 56 // depth >= 128 will never happen, storage leaves will be included in their parents for i := 0; i < 14 && i < len(req.path); i++ { prio |= int64(15-req.path[i]) << (52 - i*4) // 15-nibble => lexicographic order @@ -343,13 +380,13 @@ func (s *Sync) schedule(req *request) { // children retrieves all the missing children of a state trie entry for future // retrieval scheduling. -func (s *Sync) children(req *request, object node) ([]*request, error) { +func (s *Sync) children(req *nodeRequest, object node) ([]*nodeRequest, error) { // Gather all the children of the node, irrelevant whether known or not - type child struct { + type childNode struct { path []byte node node } - var children []child + var children []childNode switch node := (object).(type) { case *shortNode: @@ -357,14 +394,14 @@ func (s *Sync) children(req *request, object node) ([]*request, error) { if hasTerm(key) { key = key[:len(key)-1] } - children = []child{{ + children = []childNode{{ node: node.Val, path: append(append([]byte(nil), req.path...), key...), }} case *fullNode: for i := 0; i < 17; i++ { if node.Children[i] != nil { - children = append(children, child{ + children = append(children, childNode{ node: node.Children[i], path: append(append([]byte(nil), req.path...), byte(i)), }) @@ -374,7 +411,10 @@ func (s *Sync) children(req *request, object node) ([]*request, error) { panic(fmt.Sprintf("unknown node: %+v", node)) } // Iterate over the children, and request all unknown ones - requests := make([]*request, 0, len(children)) + var ( + missing = make(chan *nodeRequest, len(children)) + pending sync.WaitGroup + ) for _, child := range children { // Notify any external watcher of a new key/value node if req.callback != nil { @@ -386,7 +426,7 @@ func (s *Sync) children(req *request, object node) ([]*request, error) { paths = append(paths, hexToKeybytes(child.path[:2*common.HashLength])) paths = append(paths, hexToKeybytes(child.path[2*common.HashLength:])) } - if err := req.callback(paths, child.path, node, req.hash); err != nil { + if err := req.callback(paths, child.path, node, req.hash, req.path); err != nil { return nil, err } } @@ -394,22 +434,39 @@ func (s *Sync) children(req *request, object node) ([]*request, error) { // If the child references another node, resolve or schedule if node, ok := (child.node).(hashNode); ok { // Try to resolve the node from the local database - hash := common.BytesToHash(node) - if s.membatch.hasNode(hash) { - continue - } - // If database says duplicate, then at least the trie node is present - // and we hold the assumption that it's NOT legacy contract code. - if rawdb.HasTrieNode(s.database, hash) { + if s.membatch.hasNode(child.path) { continue } - // Locally unknown node, schedule for retrieval - requests = append(requests, &request{ - path: child.path, - hash: hash, - parents: []*request{req}, - callback: req.callback, - }) + // Check the presence of children concurrently + pending.Add(1) + go func(child childNode) { + defer pending.Done() + + // If database says duplicate, then at least the trie node is present + // and we hold the assumption that it's NOT legacy contract code. + chash := common.BytesToHash(node) + if rawdb.HasTrieNode(s.database, chash) { + return + } + // Locally unknown node, schedule for retrieval + missing <- &nodeRequest{ + path: child.path, + hash: chash, + parent: req, + callback: req.callback, + } + }(child) + } + } + pending.Wait() + + requests := make([]*nodeRequest, 0, len(children)) + for done := false; !done; { + select { + case miss := <-missing: + requests = append(requests, miss) + default: + done = true } } return requests, nil @@ -418,22 +475,40 @@ func (s *Sync) children(req *request, object node) ([]*request, error) { // commit finalizes a retrieval request and stores it into the membatch. If any // of the referencing parent requests complete due to this commit, they are also // committed themselves. -func (s *Sync) commit(req *request) (err error) { +func (s *Sync) commitNodeRequest(req *nodeRequest) error { // Write the node content to the membatch - if req.code { - s.membatch.codes[req.hash] = req.data - delete(s.codeReqs, req.hash) - s.fetches[len(req.path)]-- - } else { - s.membatch.nodes[req.hash] = req.data - delete(s.nodeReqs, req.hash) - s.fetches[len(req.path)]-- + s.membatch.nodes[string(req.path)] = req.data + s.membatch.hashes[string(req.path)] = req.hash + + delete(s.nodeReqs, string(req.path)) + s.fetches[len(req.path)]-- + + // Check parent for completion + if req.parent != nil { + req.parent.deps-- + if req.parent.deps == 0 { + if err := s.commitNodeRequest(req.parent); err != nil { + return err + } + } } + return nil +} + +// commit finalizes a retrieval request and stores it into the membatch. If any +// of the referencing parent requests complete due to this commit, they are also +// committed themselves. +func (s *Sync) commitCodeRequest(req *codeRequest) error { + // Write the node content to the membatch + s.membatch.codes[req.hash] = req.data + delete(s.codeReqs, req.hash) + s.fetches[len(req.path)]-- + // Check all parents for completion for _, parent := range req.parents { parent.deps-- if parent.deps == 0 { - if err := s.commit(parent); err != nil { + if err := s.commitNodeRequest(parent); err != nil { return err } } diff --git a/trie/sync_test.go b/trie/sync_test.go index 970730b671870..9fd1d636c0367 100644 --- a/trie/sync_test.go +++ b/trie/sync_test.go @@ -18,6 +18,7 @@ package trie import ( "bytes" + "fmt" "testing" "github.com/ethereum/go-ethereum/common" @@ -26,10 +27,10 @@ import ( ) // makeTestTrie create a sample test trie to test node-wise reconstruction. -func makeTestTrie() (*Database, *SecureTrie, map[string][]byte) { +func makeTestTrie() (*Database, *StateTrie, map[string][]byte) { // Create an empty trie triedb := NewDatabase(memorydb.New()) - trie, _ := NewSecure(common.Hash{}, triedb) + trie, _ := NewStateTrie(common.Hash{}, common.Hash{}, triedb) // Fill it with some arbitrary data content := make(map[string][]byte) @@ -50,9 +51,15 @@ func makeTestTrie() (*Database, *SecureTrie, map[string][]byte) { trie.Update(key, val) } } - trie.Commit(nil) - - // Return the generated trie + root, nodes, err := trie.Commit(false) + if err != nil { + panic(fmt.Errorf("failed to commit trie %v", err)) + } + if err := triedb.Update(NewWithNodeSet(nodes)); err != nil { + panic(fmt.Errorf("failed to commit db %v", err)) + } + // Re-create the trie based on the new state + trie, _ = NewSecure(common.Hash{}, root, triedb) return triedb, trie, content } @@ -60,7 +67,7 @@ func makeTestTrie() (*Database, *SecureTrie, map[string][]byte) { // content map. func checkTrieContents(t *testing.T, db *Database, root []byte, content map[string][]byte) { // Check root availability and trie contents - trie, err := NewSecure(common.BytesToHash(root), db) + trie, err := NewStateTrie(common.Hash{}, common.BytesToHash(root), db) if err != nil { t.Fatalf("failed to create trie at %x: %v", root, err) } @@ -77,7 +84,7 @@ func checkTrieContents(t *testing.T, db *Database, root []byte, content map[stri // checkTrieConsistency checks that all nodes in a trie are indeed present. func checkTrieConsistency(db *Database, root common.Hash) error { // Create and iterate a trie rooted in a subnode - trie, err := NewSecure(root, db) + trie, err := NewStateTrie(common.Hash{}, root, db) if err != nil { return nil // Consider a non existent state consistent } @@ -87,17 +94,24 @@ func checkTrieConsistency(db *Database, root common.Hash) error { return it.Error() } +// trieElement represents the element in the state trie(bytecode or trie node). +type trieElement struct { + path string + hash common.Hash + syncPath SyncPath +} + // Tests that an empty trie is not scheduled for syncing. func TestEmptySync(t *testing.T) { dbA := NewDatabase(memorydb.New()) dbB := NewDatabase(memorydb.New()) - emptyA, _ := New(common.Hash{}, dbA) - emptyB, _ := New(emptyRoot, dbB) + emptyA := NewEmpty(dbA) + emptyB, _ := New(common.Hash{}, emptyRoot, dbB) for i, trie := range []*Trie{emptyA, emptyB} { sync := NewSync(trie.Hash(), memorydb.New(), nil) - if nodes, paths, codes := sync.Missing(1); len(nodes) != 0 || len(paths) != 0 || len(codes) != 0 { - t.Errorf("test %d: content requested for empty trie: %v, %v, %v", i, nodes, paths, codes) + if paths, nodes, codes := sync.Missing(1); len(paths) != 0 || len(nodes) != 0 || len(codes) != 0 { + t.Errorf("test %d: content requested for empty trie: %v, %v, %v", i, paths, nodes, codes) } } } @@ -118,35 +132,38 @@ func testIterativeSync(t *testing.T, count int, bypath bool) { triedb := NewDatabase(diskdb) sched := NewSync(srcTrie.Hash(), diskdb, nil) - nodes, paths, codes := sched.Missing(count) - var ( - hashQueue []common.Hash - pathQueue []SyncPath - ) - if !bypath { - hashQueue = append(append(hashQueue[:0], nodes...), codes...) - } else { - hashQueue = append(hashQueue[:0], codes...) - pathQueue = append(pathQueue[:0], paths...) + // The code requests are ignored here since there is no code + // at the testing trie. + paths, nodes, _ := sched.Missing(count) + var elements []trieElement + for i := 0; i < len(paths); i++ { + elements = append(elements, trieElement{ + path: paths[i], + hash: nodes[i], + syncPath: NewSyncPath([]byte(paths[i])), + }) } - for len(hashQueue)+len(pathQueue) > 0 { - results := make([]SyncResult, len(hashQueue)+len(pathQueue)) - for i, hash := range hashQueue { - data, err := srcDb.Node(hash) - if err != nil { - t.Fatalf("failed to retrieve node data for hash %x: %v", hash, err) + for len(elements) > 0 { + results := make([]NodeSyncResult, len(elements)) + if !bypath { + for i, element := range elements { + data, err := srcDb.Node(element.hash) + if err != nil { + t.Fatalf("failed to retrieve node data for hash %x: %v", element.hash, err) + } + results[i] = NodeSyncResult{element.path, data} } - results[i] = SyncResult{hash, data} - } - for i, path := range pathQueue { - data, _, err := srcTrie.TryGetNode(path[0]) - if err != nil { - t.Fatalf("failed to retrieve node data for path %x: %v", path, err) + } else { + for i, element := range elements { + data, _, err := srcTrie.TryGetNode(element.syncPath[len(element.syncPath)-1]) + if err != nil { + t.Fatalf("failed to retrieve node data for path %x: %v", element.path, err) + } + results[i] = NodeSyncResult{element.path, data} } - results[len(hashQueue)+i] = SyncResult{crypto.Keccak256Hash(data), data} } for _, result := range results { - if err := sched.Process(result); err != nil { + if err := sched.ProcessNode(result); err != nil { t.Fatalf("failed to process result %v", err) } } @@ -156,12 +173,14 @@ func testIterativeSync(t *testing.T, count int, bypath bool) { } batch.Write() - nodes, paths, codes = sched.Missing(count) - if !bypath { - hashQueue = append(append(hashQueue[:0], nodes...), codes...) - } else { - hashQueue = append(hashQueue[:0], codes...) - pathQueue = append(pathQueue[:0], paths...) + paths, nodes, _ = sched.Missing(count) + elements = elements[:0] + for i := 0; i < len(paths); i++ { + elements = append(elements, trieElement{ + path: paths[i], + hash: nodes[i], + syncPath: NewSyncPath([]byte(paths[i])), + }) } } // Cross check that the two tries are in sync @@ -179,21 +198,29 @@ func TestIterativeDelayedSync(t *testing.T) { triedb := NewDatabase(diskdb) sched := NewSync(srcTrie.Hash(), diskdb, nil) - nodes, _, codes := sched.Missing(10000) - queue := append(append([]common.Hash{}, nodes...), codes...) - - for len(queue) > 0 { + // The code requests are ignored here since there is no code + // at the testing trie. + paths, nodes, _ := sched.Missing(10000) + var elements []trieElement + for i := 0; i < len(paths); i++ { + elements = append(elements, trieElement{ + path: paths[i], + hash: nodes[i], + syncPath: NewSyncPath([]byte(paths[i])), + }) + } + for len(elements) > 0 { // Sync only half of the scheduled nodes - results := make([]SyncResult, len(queue)/2+1) - for i, hash := range queue[:len(results)] { - data, err := srcDb.Node(hash) + results := make([]NodeSyncResult, len(elements)/2+1) + for i, element := range elements[:len(results)] { + data, err := srcDb.Node(element.hash) if err != nil { - t.Fatalf("failed to retrieve node data for %x: %v", hash, err) + t.Fatalf("failed to retrieve node data for %x: %v", element.hash, err) } - results[i] = SyncResult{hash, data} + results[i] = NodeSyncResult{element.path, data} } for _, result := range results { - if err := sched.Process(result); err != nil { + if err := sched.ProcessNode(result); err != nil { t.Fatalf("failed to process result %v", err) } } @@ -203,8 +230,15 @@ func TestIterativeDelayedSync(t *testing.T) { } batch.Write() - nodes, _, codes = sched.Missing(10000) - queue = append(append(queue[len(results):], nodes...), codes...) + paths, nodes, _ = sched.Missing(10000) + elements = elements[len(results):] + for i := 0; i < len(paths); i++ { + elements = append(elements, trieElement{ + path: paths[i], + hash: nodes[i], + syncPath: NewSyncPath([]byte(paths[i])), + }) + } } // Cross check that the two tries are in sync checkTrieContents(t, triedb, srcTrie.Hash().Bytes(), srcData) @@ -225,24 +259,30 @@ func testIterativeRandomSync(t *testing.T, count int) { triedb := NewDatabase(diskdb) sched := NewSync(srcTrie.Hash(), diskdb, nil) - queue := make(map[common.Hash]struct{}) - nodes, _, codes := sched.Missing(count) - for _, hash := range append(nodes, codes...) { - queue[hash] = struct{}{} + // The code requests are ignored here since there is no code + // at the testing trie. + paths, nodes, _ := sched.Missing(count) + queue := make(map[string]trieElement) + for i, path := range paths { + queue[path] = trieElement{ + path: paths[i], + hash: nodes[i], + syncPath: NewSyncPath([]byte(paths[i])), + } } for len(queue) > 0 { // Fetch all the queued nodes in a random order - results := make([]SyncResult, 0, len(queue)) - for hash := range queue { - data, err := srcDb.Node(hash) + results := make([]NodeSyncResult, 0, len(queue)) + for path, element := range queue { + data, err := srcDb.Node(element.hash) if err != nil { - t.Fatalf("failed to retrieve node data for %x: %v", hash, err) + t.Fatalf("failed to retrieve node data for %x: %v", element.hash, err) } - results = append(results, SyncResult{hash, data}) + results = append(results, NodeSyncResult{path, data}) } // Feed the retrieved results back and queue new tasks for _, result := range results { - if err := sched.Process(result); err != nil { + if err := sched.ProcessNode(result); err != nil { t.Fatalf("failed to process result %v", err) } } @@ -252,10 +292,14 @@ func testIterativeRandomSync(t *testing.T, count int) { } batch.Write() - queue = make(map[common.Hash]struct{}) - nodes, _, codes = sched.Missing(count) - for _, hash := range append(nodes, codes...) { - queue[hash] = struct{}{} + paths, nodes, _ = sched.Missing(count) + queue = make(map[string]trieElement) + for i, path := range paths { + queue[path] = trieElement{ + path: path, + hash: nodes[i], + syncPath: NewSyncPath([]byte(path)), + } } } // Cross check that the two tries are in sync @@ -273,20 +317,26 @@ func TestIterativeRandomDelayedSync(t *testing.T) { triedb := NewDatabase(diskdb) sched := NewSync(srcTrie.Hash(), diskdb, nil) - queue := make(map[common.Hash]struct{}) - nodes, _, codes := sched.Missing(10000) - for _, hash := range append(nodes, codes...) { - queue[hash] = struct{}{} + // The code requests are ignored here since there is no code + // at the testing trie. + paths, nodes, _ := sched.Missing(10000) + queue := make(map[string]trieElement) + for i, path := range paths { + queue[path] = trieElement{ + path: path, + hash: nodes[i], + syncPath: NewSyncPath([]byte(path)), + } } for len(queue) > 0 { // Sync only half of the scheduled nodes, even those in random order - results := make([]SyncResult, 0, len(queue)/2+1) - for hash := range queue { - data, err := srcDb.Node(hash) + results := make([]NodeSyncResult, 0, len(queue)/2+1) + for path, element := range queue { + data, err := srcDb.Node(element.hash) if err != nil { - t.Fatalf("failed to retrieve node data for %x: %v", hash, err) + t.Fatalf("failed to retrieve node data for %x: %v", element.hash, err) } - results = append(results, SyncResult{hash, data}) + results = append(results, NodeSyncResult{path, data}) if len(results) >= cap(results) { break @@ -294,7 +344,7 @@ func TestIterativeRandomDelayedSync(t *testing.T) { } // Feed the retrieved results back and queue new tasks for _, result := range results { - if err := sched.Process(result); err != nil { + if err := sched.ProcessNode(result); err != nil { t.Fatalf("failed to process result %v", err) } } @@ -304,11 +354,15 @@ func TestIterativeRandomDelayedSync(t *testing.T) { } batch.Write() for _, result := range results { - delete(queue, result.Hash) - } - nodes, _, codes = sched.Missing(10000) - for _, hash := range append(nodes, codes...) { - queue[hash] = struct{}{} + delete(queue, result.Path) + } + paths, nodes, _ = sched.Missing(10000) + for i, path := range paths { + queue[path] = trieElement{ + path: path, + hash: nodes[i], + syncPath: NewSyncPath([]byte(path)), + } } } // Cross check that the two tries are in sync @@ -326,26 +380,35 @@ func TestDuplicateAvoidanceSync(t *testing.T) { triedb := NewDatabase(diskdb) sched := NewSync(srcTrie.Hash(), diskdb, nil) - nodes, _, codes := sched.Missing(0) - queue := append(append([]common.Hash{}, nodes...), codes...) + // The code requests are ignored here since there is no code + // at the testing trie. + paths, nodes, _ := sched.Missing(0) + var elements []trieElement + for i := 0; i < len(paths); i++ { + elements = append(elements, trieElement{ + path: paths[i], + hash: nodes[i], + syncPath: NewSyncPath([]byte(paths[i])), + }) + } requested := make(map[common.Hash]struct{}) - for len(queue) > 0 { - results := make([]SyncResult, len(queue)) - for i, hash := range queue { - data, err := srcDb.Node(hash) + for len(elements) > 0 { + results := make([]NodeSyncResult, len(elements)) + for i, element := range elements { + data, err := srcDb.Node(element.hash) if err != nil { - t.Fatalf("failed to retrieve node data for %x: %v", hash, err) + t.Fatalf("failed to retrieve node data for %x: %v", element.hash, err) } - if _, ok := requested[hash]; ok { - t.Errorf("hash %x already requested once", hash) + if _, ok := requested[element.hash]; ok { + t.Errorf("hash %x already requested once", element.hash) } - requested[hash] = struct{}{} + requested[element.hash] = struct{}{} - results[i] = SyncResult{hash, data} + results[i] = NodeSyncResult{element.path, data} } for _, result := range results { - if err := sched.Process(result); err != nil { + if err := sched.ProcessNode(result); err != nil { t.Fatalf("failed to process result %v", err) } } @@ -355,8 +418,15 @@ func TestDuplicateAvoidanceSync(t *testing.T) { } batch.Write() - nodes, _, codes = sched.Missing(0) - queue = append(append(queue[:0], nodes...), codes...) + paths, nodes, _ = sched.Missing(0) + elements = elements[:0] + for i := 0; i < len(paths); i++ { + elements = append(elements, trieElement{ + path: paths[i], + hash: nodes[i], + syncPath: NewSyncPath([]byte(paths[i])), + }) + } } // Cross check that the two tries are in sync checkTrieContents(t, triedb, srcTrie.Hash().Bytes(), srcData) @@ -373,23 +443,34 @@ func TestIncompleteSync(t *testing.T) { triedb := NewDatabase(diskdb) sched := NewSync(srcTrie.Hash(), diskdb, nil) - var added []common.Hash - - nodes, _, codes := sched.Missing(1) - queue := append(append([]common.Hash{}, nodes...), codes...) - for len(queue) > 0 { + // The code requests are ignored here since there is no code + // at the testing trie. + var ( + added []common.Hash + elements []trieElement + root = srcTrie.Hash() + ) + paths, nodes, _ := sched.Missing(1) + for i := 0; i < len(paths); i++ { + elements = append(elements, trieElement{ + path: paths[i], + hash: nodes[i], + syncPath: NewSyncPath([]byte(paths[i])), + }) + } + for len(elements) > 0 { // Fetch a batch of trie nodes - results := make([]SyncResult, len(queue)) - for i, hash := range queue { - data, err := srcDb.Node(hash) + results := make([]NodeSyncResult, len(elements)) + for i, element := range elements { + data, err := srcDb.Node(element.hash) if err != nil { - t.Fatalf("failed to retrieve node data for %x: %v", hash, err) + t.Fatalf("failed to retrieve node data for %x: %v", element.hash, err) } - results[i] = SyncResult{hash, data} + results[i] = NodeSyncResult{element.path, data} } // Process each of the trie nodes for _, result := range results { - if err := sched.Process(result); err != nil { + if err := sched.ProcessNode(result); err != nil { t.Fatalf("failed to process result %v", err) } } @@ -398,27 +479,36 @@ func TestIncompleteSync(t *testing.T) { t.Fatalf("failed to commit data: %v", err) } batch.Write() + for _, result := range results { - added = append(added, result.Hash) + hash := crypto.Keccak256Hash(result.Data) + if hash != root { + added = append(added, hash) + } // Check that all known sub-tries in the synced trie are complete - if err := checkTrieConsistency(triedb, result.Hash); err != nil { + if err := checkTrieConsistency(triedb, hash); err != nil { t.Fatalf("trie inconsistent: %v", err) } } // Fetch the next batch to retrieve - nodes, _, codes = sched.Missing(1) - queue = append(append(queue[:0], nodes...), codes...) + paths, nodes, _ = sched.Missing(1) + elements = elements[:0] + for i := 0; i < len(paths); i++ { + elements = append(elements, trieElement{ + path: paths[i], + hash: nodes[i], + syncPath: NewSyncPath([]byte(paths[i])), + }) + } } // Sanity check that removing any node from the database is detected - for _, node := range added[1:] { - key := node.Bytes() - value, _ := diskdb.Get(key) - - diskdb.Delete(key) - if err := checkTrieConsistency(triedb, added[0]); err == nil { - t.Fatalf("trie inconsistency not caught, missing: %x", key) + for _, hash := range added { + value, _ := diskdb.Get(hash.Bytes()) + diskdb.Delete(hash.Bytes()) + if err := checkTrieConsistency(triedb, root); err == nil { + t.Fatalf("trie inconsistency not caught, missing: %x", hash) } - diskdb.Put(key, value) + diskdb.Put(hash.Bytes(), value) } } @@ -433,21 +523,33 @@ func TestSyncOrdering(t *testing.T) { triedb := NewDatabase(diskdb) sched := NewSync(srcTrie.Hash(), diskdb, nil) - nodes, paths, _ := sched.Missing(1) - queue := append([]common.Hash{}, nodes...) - reqs := append([]SyncPath{}, paths...) + // The code requests are ignored here since there is no code + // at the testing trie. + var ( + reqs []SyncPath + elements []trieElement + ) + paths, nodes, _ := sched.Missing(1) + for i := 0; i < len(paths); i++ { + elements = append(elements, trieElement{ + path: paths[i], + hash: nodes[i], + syncPath: NewSyncPath([]byte(paths[i])), + }) + reqs = append(reqs, NewSyncPath([]byte(paths[i]))) + } - for len(queue) > 0 { - results := make([]SyncResult, len(queue)) - for i, hash := range queue { - data, err := srcDb.Node(hash) + for len(elements) > 0 { + results := make([]NodeSyncResult, len(elements)) + for i, element := range elements { + data, err := srcDb.Node(element.hash) if err != nil { - t.Fatalf("failed to retrieve node data for %x: %v", hash, err) + t.Fatalf("failed to retrieve node data for %x: %v", element.hash, err) } - results[i] = SyncResult{hash, data} + results[i] = NodeSyncResult{element.path, data} } for _, result := range results { - if err := sched.Process(result); err != nil { + if err := sched.ProcessNode(result); err != nil { t.Fatalf("failed to process result %v", err) } } @@ -457,9 +559,16 @@ func TestSyncOrdering(t *testing.T) { } batch.Write() - nodes, paths, _ = sched.Missing(1) - queue = append(queue[:0], nodes...) - reqs = append(reqs, paths...) + paths, nodes, _ = sched.Missing(1) + elements = elements[:0] + for i := 0; i < len(paths); i++ { + elements = append(elements, trieElement{ + path: paths[i], + hash: nodes[i], + syncPath: NewSyncPath([]byte(paths[i])), + }) + reqs = append(reqs, NewSyncPath([]byte(paths[i]))) + } } // Cross check that the two tries are in sync checkTrieContents(t, triedb, srcTrie.Hash().Bytes(), srcData) diff --git a/trie/trie.go b/trie/trie.go index fe7d6dc17e79f..9274d88380cc4 100644 --- a/trie/trie.go +++ b/trie/trie.go @@ -21,14 +21,10 @@ import ( "bytes" "errors" "fmt" - "sync" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/rlp" ) var ( @@ -42,35 +38,41 @@ var ( // LeafCallback is a callback type invoked when a trie operation reaches a leaf // node. // -// The paths is a path tuple identifying a particular trie node either in a single -// trie (account) or a layered trie (account -> storage). Each path in the tuple +// The keys is a path tuple identifying a particular trie node either in a single +// trie (account) or a layered trie (account -> storage). Each key in the tuple // is in the raw format(32 bytes). // -// The hexpath is a composite hexary path identifying the trie node. All the key +// The path is a composite hexary path identifying the trie node. All the key // bytes are converted to the hexary nibbles and composited with the parent path // if the trie node is in a layered trie. // // It's used by state sync and commit to allow handling external references // between account and storage tries. And also it's used in the state healing // for extracting the raw states(leaf nodes) with corresponding paths. -type LeafCallback func(paths [][]byte, hexpath []byte, leaf []byte, parent common.Hash) error +type LeafCallback func(keys [][]byte, path []byte, leaf []byte, parent common.Hash, parentPath []byte) error -// Trie is a Merkle Patricia Trie. -// The zero value is an empty trie with no database. -// Use New to create a trie that sits on top of a database. +// Trie is a Merkle Patricia Trie. Use New to create a trie that sits on +// top of a database. Whenever trie performs a commit operation, the generated +// nodes will be gathered and returned in a set. Once the trie is committed, +// it's not usable anymore. Callers have to re-create the trie with new root +// based on the updated trie database. // // Trie is not safe for concurrent use. type Trie struct { - db *Database - root node + root node + owner common.Hash // Keep track of the number leaves which have been inserted since the last // hashing operation. This number will not directly map to the number of - // actually unhashed nodes + // actually unhashed nodes. unhashed int - // tracer is the state diff tracer can be used to track newly added/deleted - // trie node. It will be reset after each commit operation. + // db is the handler trie can retrieve nodes from. It's + // only for reading purpose and not available for writing. + db *Database + + // tracer is the tool to track the trie changes. + // It will be reset after each commit operation. tracer *tracer } @@ -82,25 +84,25 @@ func (t *Trie) newFlag() nodeFlag { // Copy returns a copy of Trie. func (t *Trie) Copy() *Trie { return &Trie{ - db: t.db, root: t.root, + owner: t.owner, unhashed: t.unhashed, + db: t.db, tracer: t.tracer.copy(), } } -// New creates a trie with an existing root node from db. +// New creates a trie with an existing root node from db and an assigned +// owner for storage proximity. // // If root is the zero hash or the sha3 hash of an empty string, the // trie is initially empty and does not require a database. Otherwise, // New will panic if db is nil and returns a MissingNodeError if root does // not exist in the database. Accessing the trie loads nodes from db on demand. -func New(root common.Hash, db *Database) (*Trie, error) { - if db == nil { - panic("trie.New called without a database") - } +func New(owner common.Hash, root common.Hash, db *Database) (*Trie, error) { trie := &Trie{ - db: db, + owner: owner, + db: db, //tracer: newTracer(), } if root != (common.Hash{}) && root != emptyRoot { @@ -113,14 +115,10 @@ func New(root common.Hash, db *Database) (*Trie, error) { return trie, nil } -// newWithRootNode initializes the trie with the given root node. -// It's only used by range prover. -func newWithRootNode(root node) *Trie { - return &Trie{ - root: root, - //tracer: newTracer(), - db: NewDatabase(rawdb.NewMemoryDatabase()), - } +// NewEmpty is a shortcut to create empty tree. It's mostly used in tests. +func NewEmpty(db *Database) *Trie { + tr, _ := New(common.Hash{}, common.Hash{}, db) + return tr } // NodeIterator returns an iterator that returns nodes of the trie. Iteration starts at @@ -275,14 +273,6 @@ func (t *Trie) Update(key, value []byte) { } } -func (t *Trie) TryUpdateAccount(key []byte, acc *types.StateAccount) error { - data, err := rlp.EncodeToBytes(acc) - if err != nil { - return fmt.Errorf("can't encode object at %x: %w", key[:], err) - } - return t.TryUpdate(key, data) -} - // TryUpdate associates key with value in the trie. Subsequent calls to // Get will return value. If value has length zero, any existing value // is deleted from the trie and calls to Get will return nil. @@ -292,6 +282,12 @@ func (t *Trie) TryUpdateAccount(key []byte, acc *types.StateAccount) error { // // If a node was not found in the database, a MissingNodeError is returned. func (t *Trie) TryUpdate(key, value []byte) error { + return t.tryUpdate(key, value) +} + +// tryUpdate expects an RLP-encoded value and performs the core function +// for TryUpdate and TryUpdateAccount. +func (t *Trie) tryUpdate(key, value []byte) error { t.unhashed++ k := keybytesToHex(key) if len(value) != 0 { @@ -497,7 +493,7 @@ func (t *Trie) delete(n node, prefix, key []byte) (bool, node, error) { // shortNode{..., shortNode{...}}. Since the entry // might not be loaded yet, resolve it just for this // check. - cnode, err := t.resolve(n.Children[pos], prefix) + cnode, err := t.resolve(n.Children[pos], append(prefix, byte(pos))) if err != nil { return false, nil, err } @@ -557,21 +553,25 @@ func (t *Trie) resolve(n node, prefix []byte) (node, error) { return n, nil } +// resolveHash loads node from the underlying database with the provided +// node hash and path prefix. func (t *Trie) resolveHash(n hashNode, prefix []byte) (node, error) { hash := common.BytesToHash(n) if node := t.db.node(hash); node != nil { return node, nil } - return nil, &MissingNodeError{NodeHash: hash, Path: prefix} + return nil, &MissingNodeError{Owner: t.owner, NodeHash: hash, Path: prefix} } +// resolveHash loads rlp-encoded node blob from the underlying database +// with the provided node hash and path prefix. func (t *Trie) resolveBlob(n hashNode, prefix []byte) ([]byte, error) { hash := common.BytesToHash(n) blob, _ := t.db.Node(hash) if len(blob) != 0 { return blob, nil } - return nil, &MissingNodeError{NodeHash: hash, Path: prefix} + return nil, &MissingNodeError{Owner: t.owner, NodeHash: hash, Path: prefix} } // Hash returns the root hash of the trie. It does not write to the @@ -582,53 +582,37 @@ func (t *Trie) Hash() common.Hash { return common.BytesToHash(hash.(hashNode)) } -// Commit writes all nodes to the trie's memory database, tracking the internal -// and external (for account tries) references. -func (t *Trie) Commit(onleaf LeafCallback) (common.Hash, int, error) { - if t.db == nil { - panic("commit called on trie with nil database") - } +// Commit collects all dirty nodes in the trie and replace them with the +// corresponding node hash. All collected nodes(including dirty leaves if +// collectLeaf is true) will be encapsulated into a nodeset for return. +// The returned nodeset can be nil if the trie is clean(nothing to commit). +// Once the trie is committed, it's not usable anymore. A new trie must +// be created with new root and updated trie database for following usage +func (t *Trie) Commit(collectLeaf bool) (common.Hash, *NodeSet, error) { defer t.tracer.reset() if t.root == nil { - return emptyRoot, 0, nil + return emptyRoot, nil, nil } // Derive the hash for all dirty nodes first. We hold the assumption // in the following procedure that all nodes are hashed. rootHash := t.Hash() - h := newCommitter() - defer returnCommitterToPool(h) - - // Do a quick check if we really need to commit, before we spin - // up goroutines. This can happen e.g. if we load a trie for reading storage - // values, but don't write to it. - if _, dirty := t.root.cache(); !dirty { - return rootHash, 0, nil - } - var wg sync.WaitGroup - if onleaf != nil { - h.onleaf = onleaf - h.leafCh = make(chan *leaf, leafChanSize) - wg.Add(1) - go func() { - defer wg.Done() - h.commitLoop(t.db) - }() - } - newRoot, committed, err := h.Commit(t.root, t.db) - if onleaf != nil { - // The leafch is created in newCommitter if there was an onleaf callback - // provided. The commitLoop only _reads_ from it, and the commit - // operation was the sole writer. Therefore, it's safe to close this - // channel here. - close(h.leafCh) - wg.Wait() + + // Do a quick check if we really need to commit. This can happen e.g. + // if we load a trie for reading storage values, but don't write to it. + if hashedNode, dirty := t.root.cache(); !dirty { + // Replace the root node with the origin hash in order to + // ensure all resolved nodes are dropped after the commit. + t.root = hashedNode + return rootHash, nil, nil } + h := newCommitter(t.owner, collectLeaf) + newRoot, nodes, err := h.Commit(t.root) if err != nil { - return common.Hash{}, 0, err + return common.Hash{}, nil, err } t.root = newRoot - return rootHash, committed, nil + return rootHash, nodes, nil } // hashRoot calculates the root hash of the given trie @@ -647,6 +631,8 @@ func (t *Trie) hashRoot() (node, node, error) { // Reset drops the referenced root node and cleans all internal state. func (t *Trie) Reset() { t.root = nil + t.owner = common.Hash{} t.unhashed = 0 + //t.db = nil t.tracer.reset() } diff --git a/trie/trie_test.go b/trie/trie_test.go index f994e31af40e1..3e29600bbd121 100644 --- a/trie/trie_test.go +++ b/trie/trie_test.go @@ -24,7 +24,6 @@ import ( "hash" "math/big" "math/rand" - "os" "reflect" "testing" "testing/quick" @@ -35,7 +34,6 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/ethdb/leveldb" "github.com/ethereum/go-ethereum/ethdb/memorydb" "github.com/ethereum/go-ethereum/rlp" "golang.org/x/crypto/sha3" @@ -46,14 +44,8 @@ func init() { spew.Config.DisableMethods = false } -// Used for testing -func newEmpty() *Trie { - trie, _ := New(common.Hash{}, NewDatabase(memorydb.New())) - return trie -} - func TestEmptyTrie(t *testing.T) { - trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) res := trie.Hash() exp := emptyRoot if res != exp { @@ -62,7 +54,7 @@ func TestEmptyTrie(t *testing.T) { } func TestNull(t *testing.T) { - trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) key := make([]byte, 32) value := []byte("test") trie.Update(key, value) @@ -72,7 +64,7 @@ func TestNull(t *testing.T) { } func TestMissingRoot(t *testing.T) { - trie, err := New(common.HexToHash("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33"), NewDatabase(memorydb.New())) + trie, err := New(common.Hash{}, common.HexToHash("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33"), NewDatabase(memorydb.New())) if trie != nil { t.Error("New returned non-nil trie for invalid root") } @@ -88,35 +80,36 @@ func testMissingNode(t *testing.T, memonly bool) { diskdb := memorydb.New() triedb := NewDatabase(diskdb) - trie, _ := New(common.Hash{}, triedb) + trie := NewEmpty(triedb) updateString(trie, "120000", "qwerqwerqwerqwerqwerqwerqwerqwer") updateString(trie, "123456", "asdfasdfasdfasdfasdfasdfasdfasdf") - root, _, _ := trie.Commit(nil) + root, nodes, _ := trie.Commit(false) + triedb.Update(NewWithNodeSet(nodes)) if !memonly { triedb.Commit(root, true, nil) } - trie, _ = New(root, triedb) + trie, _ = New(common.Hash{}, root, triedb) _, err := trie.TryGet([]byte("120000")) if err != nil { t.Errorf("Unexpected error: %v", err) } - trie, _ = New(root, triedb) + trie, _ = New(common.Hash{}, root, triedb) _, err = trie.TryGet([]byte("120099")) if err != nil { t.Errorf("Unexpected error: %v", err) } - trie, _ = New(root, triedb) + trie, _ = New(common.Hash{}, root, triedb) _, err = trie.TryGet([]byte("123456")) if err != nil { t.Errorf("Unexpected error: %v", err) } - trie, _ = New(root, triedb) + trie, _ = New(common.Hash{}, root, triedb) err = trie.TryUpdate([]byte("120099"), []byte("zxcvzxcvzxcvzxcvzxcvzxcvzxcvzxcv")) if err != nil { t.Errorf("Unexpected error: %v", err) } - trie, _ = New(root, triedb) + trie, _ = New(common.Hash{}, root, triedb) err = trie.TryDelete([]byte("123456")) if err != nil { t.Errorf("Unexpected error: %v", err) @@ -129,27 +122,27 @@ func testMissingNode(t *testing.T, memonly bool) { diskdb.Delete(hash[:]) } - trie, _ = New(root, triedb) + trie, _ = New(common.Hash{}, root, triedb) _, err = trie.TryGet([]byte("120000")) if _, ok := err.(*MissingNodeError); !ok { t.Errorf("Wrong error: %v", err) } - trie, _ = New(root, triedb) + trie, _ = New(common.Hash{}, root, triedb) _, err = trie.TryGet([]byte("120099")) if _, ok := err.(*MissingNodeError); !ok { t.Errorf("Wrong error: %v", err) } - trie, _ = New(root, triedb) + trie, _ = New(common.Hash{}, root, triedb) _, err = trie.TryGet([]byte("123456")) if err != nil { t.Errorf("Unexpected error: %v", err) } - trie, _ = New(root, triedb) + trie, _ = New(common.Hash{}, root, triedb) err = trie.TryUpdate([]byte("120099"), []byte("zxcv")) if _, ok := err.(*MissingNodeError); !ok { t.Errorf("Wrong error: %v", err) } - trie, _ = New(root, triedb) + trie, _ = New(common.Hash{}, root, triedb) err = trie.TryDelete([]byte("123456")) if _, ok := err.(*MissingNodeError); !ok { t.Errorf("Wrong error: %v", err) @@ -157,7 +150,7 @@ func testMissingNode(t *testing.T, memonly bool) { } func TestInsert(t *testing.T) { - trie := newEmpty() + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) updateString(trie, "doe", "reindeer") updateString(trie, "dog", "puppy") @@ -169,11 +162,11 @@ func TestInsert(t *testing.T) { t.Errorf("case 1: exp %x got %x", exp, root) } - trie = newEmpty() + trie = NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) updateString(trie, "A", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") exp = common.HexToHash("d23786fb4a010da3ce639d66d5e904a11dbc02746d1ce25029e53290cabf28ab") - root, _, err := trie.Commit(nil) + root, _, err := trie.Commit(false) if err != nil { t.Fatalf("commit error: %v", err) } @@ -183,7 +176,8 @@ func TestInsert(t *testing.T) { } func TestGet(t *testing.T) { - trie := newEmpty() + db := NewDatabase(rawdb.NewMemoryDatabase()) + trie := NewEmpty(db) updateString(trie, "doe", "reindeer") updateString(trie, "dog", "puppy") updateString(trie, "dogglesworth", "cat") @@ -193,21 +187,21 @@ func TestGet(t *testing.T) { if !bytes.Equal(res, []byte("puppy")) { t.Errorf("expected puppy got %x", res) } - unknown := getString(trie, "unknown") if unknown != nil { t.Errorf("expected nil got %x", unknown) } - if i == 1 { return } - trie.Commit(nil) + root, nodes, _ := trie.Commit(false) + db.Update(NewWithNodeSet(nodes)) + trie, _ = New(common.Hash{}, root, db) } } func TestDelete(t *testing.T) { - trie := newEmpty() + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) vals := []struct{ k, v string }{ {"do", "verb"}, {"ether", "wookiedoo"}, @@ -234,7 +228,7 @@ func TestDelete(t *testing.T) { } func TestEmptyValues(t *testing.T) { - trie := newEmpty() + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) vals := []struct{ k, v string }{ {"do", "verb"}, @@ -258,7 +252,8 @@ func TestEmptyValues(t *testing.T) { } func TestReplication(t *testing.T) { - trie := newEmpty() + triedb := NewDatabase(rawdb.NewMemoryDatabase()) + trie := NewEmpty(triedb) vals := []struct{ k, v string }{ {"do", "verb"}, {"ether", "wookiedoo"}, @@ -271,13 +266,14 @@ func TestReplication(t *testing.T) { for _, val := range vals { updateString(trie, val.k, val.v) } - exp, _, err := trie.Commit(nil) + exp, nodes, err := trie.Commit(false) if err != nil { t.Fatalf("commit error: %v", err) } + triedb.Update(NewWithNodeSet(nodes)) // create a new trie on top of the database and check that lookups work. - trie2, err := New(exp, trie.db) + trie2, err := New(common.Hash{}, exp, triedb) if err != nil { t.Fatalf("can't recreate trie at %x: %v", exp, err) } @@ -286,7 +282,7 @@ func TestReplication(t *testing.T) { t.Errorf("trie2 doesn't have %q => %q", kv.k, kv.v) } } - hash, _, err := trie2.Commit(nil) + hash, nodes, err := trie2.Commit(false) if err != nil { t.Fatalf("commit error: %v", err) } @@ -294,6 +290,14 @@ func TestReplication(t *testing.T) { t.Errorf("root failure. expected %x got %x", exp, hash) } + // recreate the trie after commit + if nodes != nil { + triedb.Update(NewWithNodeSet(nodes)) + } + trie2, err = New(common.Hash{}, hash, triedb) + if err != nil { + t.Fatalf("can't recreate trie at %x: %v", exp, err) + } // perform some insertions on the new trie. vals2 := []struct{ k, v string }{ {"do", "verb"}, @@ -315,7 +319,7 @@ func TestReplication(t *testing.T) { } func TestLargeValue(t *testing.T) { - trie := newEmpty() + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) trie.Update([]byte("key1"), []byte{99, 99, 99, 99}) trie.Update([]byte("key2"), bytes.Repeat([]byte{1}, 32)) trie.Hash() @@ -352,7 +356,6 @@ func TestRandomCases(t *testing.T) { {op: 1, key: common.Hex2Bytes("fd"), value: common.Hex2Bytes("")}, // step 25 } runRandTest(rt) - } // randTest performs random trie operations. @@ -370,9 +373,8 @@ const ( opUpdate = iota opDelete opGet - opCommit opHash - opReset + opCommit opItercheckhash opNodeDiff opMax // boundary value, not an actual op @@ -410,10 +412,10 @@ func (randTest) Generate(r *rand.Rand, size int) reflect.Value { func runRandTest(rt randTest) bool { var ( - triedb = NewDatabase(memorydb.New()) - tr, _ = New(common.Hash{}, triedb) - values = make(map[string]string) // tracks content of the trie - origTrie, _ = New(common.Hash{}, triedb) + triedb = NewDatabase(memorydb.New()) + tr = NewEmpty(triedb) + values = make(map[string]string) // tracks content of the trie + origTrie = NewEmpty(triedb) ) tr.tracer = newTracer() @@ -432,20 +434,20 @@ func runRandTest(rt randTest) bool { v := tr.Get(step.key) want := values[string(step.key)] if string(v) != want { - rt[i].err = fmt.Errorf("mismatch for key 0x%x, got 0x%x want 0x%x", step.key, v, want) + rt[i].err = fmt.Errorf("mismatch for key %#x, got %#x want %#x", step.key, v, want) } - case opCommit: - _, _, rt[i].err = tr.Commit(nil) - origTrie = tr.Copy() case opHash: tr.Hash() - case opReset: - hash, _, err := tr.Commit(nil) + case opCommit: + hash, nodes, err := tr.Commit(false) if err != nil { rt[i].err = err return false } - newtr, err := New(hash, triedb) + if nodes != nil { + triedb.Update(NewWithNodeSet(nodes)) + } + newtr, err := New(common.Hash{}, hash, triedb) if err != nil { rt[i].err = err return false @@ -455,7 +457,7 @@ func runRandTest(rt randTest) bool { origTrie = tr.Copy() case opItercheckhash: - checktr, _ := New(common.Hash{}, triedb) + checktr := NewEmpty(triedb) it := NewIterator(tr.NodeIterator(nil)) for it.Next() { checktr.Update(it.Key, it.Value) @@ -534,44 +536,31 @@ func TestRandom(t *testing.T) { } } -func BenchmarkGet(b *testing.B) { benchGet(b, false) } -func BenchmarkGetDB(b *testing.B) { benchGet(b, true) } +func BenchmarkGet(b *testing.B) { benchGet(b) } func BenchmarkUpdateBE(b *testing.B) { benchUpdate(b, binary.BigEndian) } func BenchmarkUpdateLE(b *testing.B) { benchUpdate(b, binary.LittleEndian) } const benchElemCount = 20000 -func benchGet(b *testing.B, commit bool) { - trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) - if commit { - tmpdb := tempDB(b) - trie, _ = New(common.Hash{}, tmpdb) - } +func benchGet(b *testing.B) { + triedb := NewDatabase(rawdb.NewMemoryDatabase()) + trie := NewEmpty(triedb) k := make([]byte, 32) for i := 0; i < benchElemCount; i++ { binary.LittleEndian.PutUint64(k, uint64(i)) trie.Update(k, k) } binary.LittleEndian.PutUint64(k, benchElemCount/2) - if commit { - trie.Commit(nil) - } b.ResetTimer() for i := 0; i < b.N; i++ { trie.Get(k) } b.StopTimer() - - if commit { - ldb := trie.db.diskdb.(*leveldb.Database) - ldb.Close() - os.RemoveAll(ldb.Path()) - } } func benchUpdate(b *testing.B, e binary.ByteOrder) *Trie { - trie := newEmpty() + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) k := make([]byte, 32) b.ReportAllocs() for i := 0; i < b.N; i++ { @@ -601,7 +590,7 @@ func BenchmarkHash(b *testing.B) { // entries, then adding N more. addresses, accounts := makeAccounts(2 * b.N) // Insert the accounts into the trie and hash it - trie := newEmpty() + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) i := 0 for ; i < len(addresses)/2; i++ { trie.Update(crypto.Keccak256(addresses[i][:]), accounts[i]) @@ -622,22 +611,17 @@ func BenchmarkHash(b *testing.B) { // insert into the trie before measuring the hashing. func BenchmarkCommitAfterHash(b *testing.B) { b.Run("no-onleaf", func(b *testing.B) { - benchmarkCommitAfterHash(b, nil) + benchmarkCommitAfterHash(b, false) }) - var a types.StateAccount - onleaf := func(paths [][]byte, hexpath []byte, leaf []byte, parent common.Hash) error { - rlp.DecodeBytes(leaf, &a) - return nil - } b.Run("with-onleaf", func(b *testing.B) { - benchmarkCommitAfterHash(b, onleaf) + benchmarkCommitAfterHash(b, true) }) } -func benchmarkCommitAfterHash(b *testing.B, onleaf LeafCallback) { +func benchmarkCommitAfterHash(b *testing.B, collectLeaf bool) { // Make the random benchmark deterministic addresses, accounts := makeAccounts(b.N) - trie := newEmpty() + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) for i := 0; i < len(addresses); i++ { trie.Update(crypto.Keccak256(addresses[i][:]), accounts[i]) } @@ -645,13 +629,13 @@ func benchmarkCommitAfterHash(b *testing.B, onleaf LeafCallback) { trie.Hash() b.ResetTimer() b.ReportAllocs() - trie.Commit(onleaf) + trie.Commit(collectLeaf) } func TestTinyTrie(t *testing.T) { // Create a realistic account trie to hash _, accounts := makeAccounts(5) - trie := newEmpty() + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) trie.Update(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000001337"), accounts[3]) if exp, root := common.HexToHash("8c6a85a4d9fda98feff88450299e574e5378e32391f75a055d470ac0653f1005"), trie.Hash(); exp != root { t.Errorf("1: got %x, exp %x", root, exp) @@ -664,7 +648,7 @@ func TestTinyTrie(t *testing.T) { if exp, root := common.HexToHash("0608c1d1dc3905fa22204c7a0e43644831c3b6d3def0f274be623a948197e64a"), trie.Hash(); exp != root { t.Errorf("3: got %x, exp %x", root, exp) } - checktr, _ := New(common.Hash{}, trie.db) + checktr := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) it := NewIterator(trie.NodeIterator(nil)) for it.Next() { checktr.Update(it.Key, it.Value) @@ -677,19 +661,19 @@ func TestTinyTrie(t *testing.T) { func TestCommitAfterHash(t *testing.T) { // Create a realistic account trie to hash addresses, accounts := makeAccounts(1000) - trie := newEmpty() + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) for i := 0; i < len(addresses); i++ { trie.Update(crypto.Keccak256(addresses[i][:]), accounts[i]) } // Insert the accounts into the trie and hash it trie.Hash() - trie.Commit(nil) + trie.Commit(false) root := trie.Hash() exp := common.HexToHash("72f9d3f3fe1e1dd7b8936442e7642aef76371472d94319900790053c493f3fe6") if exp != root { t.Errorf("got %x, exp %x", root, exp) } - root, _, _ = trie.Commit(nil) + root, _, _ = trie.Commit(false) if exp != root { t.Errorf("got %x, exp %x", root, exp) } @@ -790,7 +774,7 @@ func TestCommitSequence(t *testing.T) { // This spongeDb is used to check the sequence of disk-db-writes s := &spongeDb{sponge: sha3.NewLegacyKeccak256()} db := NewDatabase(s) - trie, _ := New(common.Hash{}, db) + trie := NewEmpty(db) // Another sponge is used to check the callback-sequence callbackSponge := sha3.NewLegacyKeccak256() // Fill the trie with elements @@ -798,7 +782,8 @@ func TestCommitSequence(t *testing.T) { trie.Update(crypto.Keccak256(addresses[i][:]), accounts[i]) } // Flush trie -> database - root, _, _ := trie.Commit(nil) + root, nodes, _ := trie.Commit(false) + db.Update(NewWithNodeSet(nodes)) // Flush memdb -> disk (sponge) db.Commit(root, false, func(c common.Hash) { // And spongify the callback-order @@ -832,7 +817,7 @@ func TestCommitSequenceRandomBlobs(t *testing.T) { // This spongeDb is used to check the sequence of disk-db-writes s := &spongeDb{sponge: sha3.NewLegacyKeccak256()} db := NewDatabase(s) - trie, _ := New(common.Hash{}, db) + trie := NewEmpty(db) // Another sponge is used to check the callback-sequence callbackSponge := sha3.NewLegacyKeccak256() // Fill the trie with elements @@ -850,7 +835,8 @@ func TestCommitSequenceRandomBlobs(t *testing.T) { trie.Update(key, val) } // Flush trie -> database - root, _, _ := trie.Commit(nil) + root, nodes, _ := trie.Commit(false) + db.Update(NewWithNodeSet(nodes)) // Flush memdb -> disk (sponge) db.Commit(root, false, func(c common.Hash) { // And spongify the callback-order @@ -871,12 +857,12 @@ func TestCommitSequenceStackTrie(t *testing.T) { // This spongeDb is used to check the sequence of disk-db-writes s := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "a"} db := NewDatabase(s) - trie, _ := New(common.Hash{}, db) + trie := NewEmpty(db) // Another sponge is used for the stacktrie commits stackTrieSponge := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "b"} stTrie := NewStackTrie(stackTrieSponge) // Fill the trie with elements - for i := 1; i < count; i++ { + for i := 0; i < count; i++ { // For the stack trie, we need to do inserts in proper order key := make([]byte, 32) binary.BigEndian.PutUint64(key, uint64(i)) @@ -892,8 +878,9 @@ func TestCommitSequenceStackTrie(t *testing.T) { stTrie.TryUpdate(key, val) } // Flush trie -> database - root, _, _ := trie.Commit(nil) + root, nodes, _ := trie.Commit(false) // Flush memdb -> disk (sponge) + db.Update(NewWithNodeSet(nodes)) db.Commit(root, false, nil) // And flush stacktrie -> disk stRoot, err := stTrie.Commit() @@ -927,7 +914,7 @@ func TestCommitSequenceStackTrie(t *testing.T) { func TestCommitSequenceSmallRoot(t *testing.T) { s := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "a"} db := NewDatabase(s) - trie, _ := New(common.Hash{}, db) + trie := NewEmpty(db) // Another sponge is used for the stacktrie commits stackTrieSponge := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "b"} stTrie := NewStackTrie(stackTrieSponge) @@ -937,8 +924,9 @@ func TestCommitSequenceSmallRoot(t *testing.T) { trie.TryUpdate(key, []byte{0x1}) stTrie.TryUpdate(key, []byte{0x1}) // Flush trie -> database - root, _, _ := trie.Commit(nil) + root, nodes, _ := trie.Commit(false) // Flush memdb -> disk (sponge) + db.Update(NewWithNodeSet(nodes)) db.Commit(root, false, nil) // And flush stacktrie -> disk stRoot, err := stTrie.Commit() @@ -1000,7 +988,7 @@ func BenchmarkHashFixedSize(b *testing.B) { func benchmarkHashFixedSize(b *testing.B, addresses [][20]byte, accounts [][]byte) { b.ReportAllocs() - trie := newEmpty() + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) for i := 0; i < len(addresses); i++ { trie.Update(crypto.Keccak256(addresses[i][:]), accounts[i]) } @@ -1051,14 +1039,14 @@ func BenchmarkCommitAfterHashFixedSize(b *testing.B) { func benchmarkCommitAfterHashFixedSize(b *testing.B, addresses [][20]byte, accounts [][]byte) { b.ReportAllocs() - trie := newEmpty() + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) for i := 0; i < len(addresses); i++ { trie.Update(crypto.Keccak256(addresses[i][:]), accounts[i]) } // Insert the accounts into the trie and hash it trie.Hash() b.StartTimer() - trie.Commit(nil) + trie.Commit(false) b.StopTimer() } @@ -1103,26 +1091,19 @@ func BenchmarkDerefRootFixedSize(b *testing.B) { func benchmarkDerefRootFixedSize(b *testing.B, addresses [][20]byte, accounts [][]byte) { b.ReportAllocs() - trie := newEmpty() + triedb := NewDatabase(rawdb.NewMemoryDatabase()) + trie := NewEmpty(triedb) for i := 0; i < len(addresses); i++ { trie.Update(crypto.Keccak256(addresses[i][:]), accounts[i]) } h := trie.Hash() - trie.Commit(nil) + _, nodes, _ := trie.Commit(false) + triedb.Update(NewWithNodeSet(nodes)) b.StartTimer() - trie.db.Dereference(h) + triedb.Dereference(h) b.StopTimer() } -func tempDB(tb testing.TB) *Database { - dir := tb.TempDir() - diskdb, err := leveldb.New(dir, 256, 0, "", false) - if err != nil { - panic(fmt.Sprintf("can't create temporary database: %v", err)) - } - return NewDatabase(diskdb) -} - func getString(trie *Trie, k string) []byte { return trie.Get([]byte(k)) } diff --git a/trie/util_test.go b/trie/util_test.go index fadb0553b5298..252dc09e0804c 100644 --- a/trie/util_test.go +++ b/trie/util_test.go @@ -26,7 +26,7 @@ import ( // Tests if the trie diffs are tracked correctly. func TestTrieTracer(t *testing.T) { db := NewDatabase(rawdb.NewMemoryDatabase()) - trie, _ := New(common.Hash{}, db) + trie := NewEmpty(db) trie.tracer = newTracer() // Insert a batch of entries, all the nodes should be marked as inserted @@ -67,8 +67,11 @@ func TestTrieTracer(t *testing.T) { t.Fatalf("Unexpected deleted node tracked %d", len(deleted)) } - // Commit the changes - trie.Commit(nil) + // Commit the changes and re-create with new root + root, nodes, _ := trie.Commit(false) + db.Update(NewWithNodeSet(nodes)) + trie, _ = New(common.Hash{}, root, db) + trie.tracer = newTracer() // Delete all the elements, check deletion set for _, val := range vals { @@ -93,8 +96,7 @@ func TestTrieTracer(t *testing.T) { } func TestTrieTracerNoop(t *testing.T) { - db := NewDatabase(rawdb.NewMemoryDatabase()) - trie, _ := New(common.Hash{}, db) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) trie.tracer = newTracer() // Insert a batch of entries, all the nodes should be marked as inserted diff --git a/trie/utils.go b/trie/utils.go index 5f9e3ba58ecc3..7e26915041efd 100644 --- a/trie/utils.go +++ b/trie/utils.go @@ -29,24 +29,41 @@ package trie // This tool can track all of them no matter the node is embedded in its // parent or not, but valueNode is never tracked. // +// Besides, it's also used for recording the original value of the nodes +// when they are resolved from the disk. The pre-value of the nodes will +// be used to construct reverse-diffs in the future. +// // Note tracer is not thread-safe, callers should be responsible for handling // the concurrency issues by themselves. type tracer struct { insert map[string]struct{} delete map[string]struct{} + origin map[string][]byte } -// newTracer initializes trie node diff tracer. +// newTracer initializes the tracer for capturing trie changes. func newTracer() *tracer { return &tracer{ insert: make(map[string]struct{}), delete: make(map[string]struct{}), + origin: make(map[string][]byte), + } +} + +/* +// onRead tracks the newly loaded trie node and caches the rlp-encoded blob internally. +// Don't change the value outside of function since it's not deep-copied. +func (t *tracer) onRead(key []byte, val []byte) { + // Tracer isn't used right now, remove this check later. + if t == nil { + return } + t.origin[string(key)] = val } +*/ -// onInsert tracks the newly inserted trie node. If it's already -// in the deletion set(resurrected node), then just wipe it from -// the deletion set as it's untouched. +// onInsert tracks the newly inserted trie node. If it's already in the deletion set +// (resurrected node), then just wipe it from the deletion set as the "untouched". func (t *tracer) onInsert(key []byte) { // Tracer isn't used right now, remove this check later. if t == nil { @@ -100,6 +117,17 @@ func (t *tracer) deleteList() [][]byte { return ret } +/* +// getPrev returns the cached original value of the specified node. +func (t *tracer) getPrev(key []byte) []byte { + // Don't panic on uninitialized tracer, it's possible in testing. + if t == nil { + return nil + } + return t.origin[string(key)] +} +*/ + // reset clears the content tracked by tracer. func (t *tracer) reset() { // Tracer isn't used right now, remove this check later. @@ -108,6 +136,7 @@ func (t *tracer) reset() { } t.insert = make(map[string]struct{}) t.delete = make(map[string]struct{}) + t.origin = make(map[string][]byte) } // copy returns a deep copied tracer instance. @@ -119,6 +148,7 @@ func (t *tracer) copy() *tracer { var ( insert = make(map[string]struct{}) delete = make(map[string]struct{}) + origin = make(map[string][]byte) ) for key := range t.insert { insert[key] = struct{}{} @@ -126,8 +156,12 @@ func (t *tracer) copy() *tracer { for key := range t.delete { delete[key] = struct{}{} } + for key, val := range t.origin { + origin[key] = val + } return &tracer{ insert: insert, delete: delete, + origin: origin, } }