diff --git a/.github/ISSUE_TEMPLATE/bug.md b/.github/ISSUE_TEMPLATE/bug.md index 2aa2c48a600b0..45bfd986ac6e3 100644 --- a/.github/ISSUE_TEMPLATE/bug.md +++ b/.github/ISSUE_TEMPLATE/bug.md @@ -9,6 +9,7 @@ assignees: '' #### System information Geth version: `geth version` +CL client & version: e.g. lighthouse/nimbus/prysm@v1.0.0 OS & Version: Windows/Linux/OSX Commit hash : (if `develop`) @@ -27,4 +28,4 @@ Commit hash : (if `develop`) [backtrace] ```` -When submitting logs: please submit them as text and not screenshots. \ No newline at end of file +When submitting logs: please submit them as text and not screenshots. diff --git a/.golangci.yml b/.golangci.yml index 4950b98c21baf..8a054667e6d89 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -12,17 +12,29 @@ run: linters: disable-all: true enable: - - deadcode - goconst - goimports - gosimple - govet - ineffassign - misspell - # - staticcheck - unconvert - # - unused - - varcheck + - typecheck + - unused + - staticcheck + - bidichk + - durationcheck + - exportloopref + - whitespace + + # - structcheck # lots of false positives + # - errcheck #lot of false positives + # - contextcheck + # - errchkjson # lots of false positives + # - errorlint # this check crashes + # - exhaustive # silly check + # - makezero # false positives + # - nilerr # several intentional linters-settings: gofmt: @@ -33,18 +45,20 @@ linters-settings: issues: exclude-rules: - - path: crypto/blake2b/ - linters: - - deadcode - - path: crypto/bn256/cloudflare - linters: - - deadcode - - path: p2p/discv5/ - linters: - - deadcode - - path: core/vm/instructions_test.go - linters: - - goconst - - path: cmd/faucet/ + - path: crypto/bn256/cloudflare/optate.go linters: - deadcode + - staticcheck + - path: internal/build/pgp.go + text: 'SA1019: "golang.org/x/crypto/openpgp" is deprecated: this package is unmaintained except for security fixes.' + - path: core/vm/contracts.go + text: 'SA1019: "golang.org/x/crypto/ripemd160" is deprecated: RIPEMD-160 is a legacy hash and should not be used for new applications.' + - path: accounts/usbwallet/trezor.go + text: 'SA1019: "github.com/golang/protobuf/proto" is deprecated: Use the "google.golang.org/protobuf/proto" package instead.' + - path: accounts/usbwallet/trezor/ + text: 'SA1019: "github.com/golang/protobuf/proto" is deprecated: Use the "google.golang.org/protobuf/proto" package instead.' + exclude: + - 'SA1019: event.TypeMux is deprecated: use Feed' + - 'SA1019: strings.Title is deprecated' + - 'SA1019: strings.Title has been deprecated since Go 1.18 and an alternative has been available since Go 1.0: The rule Title uses for word boundaries does not handle Unicode punctuation properly. Use golang.org/x/text/cases instead.' + - 'SA1029: should not use built-in type string as key for value' diff --git a/.travis.yml b/.travis.yml index e08e271f3f120..a32b44506664b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,7 +16,7 @@ jobs: - stage: lint os: linux dist: bionic - go: 1.18.x + go: 1.19.x env: - lint git: @@ -31,7 +31,7 @@ jobs: os: linux arch: amd64 dist: bionic - go: 1.18.x + go: 1.19.x env: - docker services: @@ -48,7 +48,7 @@ jobs: os: linux arch: arm64 dist: bionic - go: 1.18.x + go: 1.19.x env: - docker services: @@ -65,7 +65,7 @@ jobs: if: type = push os: linux dist: bionic - go: 1.18.x + go: 1.19.x env: - ubuntu-ppa - GO111MODULE=on @@ -90,7 +90,7 @@ jobs: os: linux dist: bionic sudo: required - go: 1.18.x + go: 1.19.x env: - azure-linux - GO111MODULE=on @@ -162,7 +162,7 @@ jobs: - stage: build if: type = push os: osx - go: 1.18.x + go: 1.19.x env: - azure-osx - azure-ios @@ -194,7 +194,7 @@ jobs: os: linux arch: amd64 dist: bionic - go: 1.18.x + go: 1.19.x env: - GO111MODULE=on script: @@ -214,7 +214,7 @@ jobs: - stage: build os: linux dist: bionic - go: 1.17.x + go: 1.18.x env: - GO111MODULE=on script: @@ -225,7 +225,7 @@ jobs: if: type = cron os: linux dist: bionic - go: 1.18.x + go: 1.19.x env: - azure-purge - GO111MODULE=on @@ -239,7 +239,7 @@ jobs: if: type = cron os: linux dist: bionic - go: 1.18.x + go: 1.19.x env: - GO111MODULE=on script: diff --git a/Dockerfile b/Dockerfile index 70299190f90f5..143c92f27f505 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ COPY go.sum /go-ethereum/ RUN cd /go-ethereum && go mod download ADD . /go-ethereum -RUN cd /go-ethereum && go run build/ci.go install ./cmd/geth +RUN cd /go-ethereum && go run build/ci.go install -static ./cmd/geth # Pull Geth into a second stage deploy alpine container FROM alpine:latest diff --git a/Dockerfile.alltools b/Dockerfile.alltools index b11492cabc9c0..176c4592206d5 100644 --- a/Dockerfile.alltools +++ b/Dockerfile.alltools @@ -14,7 +14,7 @@ COPY go.sum /go-ethereum/ RUN cd /go-ethereum && go mod download ADD . /go-ethereum -RUN cd /go-ethereum && go run build/ci.go install +RUN cd /go-ethereum && go run build/ci.go install -static # Pull all binaries into a second stage deploy alpine container FROM alpine:latest diff --git a/README.md b/README.md index 0987200d3b9a2..5506001287feb 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ ## Go Ethereum -Official Golang implementation of the Ethereum protocol. +Official Golang execution layer implementation of the Ethereum protocol. [![API Reference]( https://camo.githubusercontent.com/915b7be44ada53c290eb157634330494ebe3e30a/68747470733a2f2f676f646f632e6f72672f6769746875622e636f6d2f676f6c616e672f6764646f3f7374617475732e737667 @@ -39,10 +39,10 @@ directory. | **`geth`** | Our main Ethereum CLI client. It is the entry point into the Ethereum network (main-, test- or private net), capable of running as a full node (default), archive node (retaining all historical state) or a light node (retrieving data live). It can be used by other processes as a gateway into the Ethereum network via JSON RPC endpoints exposed on top of HTTP, WebSocket and/or IPC transports. `geth --help` and the [CLI page](https://geth.ethereum.org/docs/interface/command-line-options) for command line options. | | `clef` | Stand-alone signing tool, which can be used as a backend signer for `geth`. | | `devp2p` | Utilities to interact with nodes on the networking layer, without running a full blockchain. | -| `abigen` | Source code generator to convert Ethereum contract definitions into easy to use, compile-time type-safe Go packages. It operates on plain [Ethereum contract ABIs](https://docs.soliditylang.org/en/develop/abi-spec.html) with expanded functionality if the contract bytecode is also available. However, it also accepts Solidity source files, making development much more streamlined. Please see our [Native DApps](https://geth.ethereum.org/docs/dapp/native-bindings) page for details. | +| `abigen` | Source code generator to convert Ethereum contract definitions into easy-to-use, compile-time type-safe Go packages. It operates on plain [Ethereum contract ABIs](https://docs.soliditylang.org/en/develop/abi-spec.html) with expanded functionality if the contract bytecode is also available. However, it also accepts Solidity source files, making development much more streamlined. Please see our [Native DApps](https://geth.ethereum.org/docs/dapp/native-bindings) page for details. | | `bootnode` | Stripped down version of our Ethereum client implementation that only takes part in the network node discovery protocol, but does not run any of the higher level application protocols. It can be used as a lightweight bootstrap node to aid in finding peers in private networks. | | `evm` | Developer utility version of the EVM (Ethereum Virtual Machine) that is capable of running bytecode snippets within a configurable environment and execution mode. Its purpose is to allow isolated, fine-grained debugging of EVM opcodes (e.g. `evm --code 60ff60ff --debug run`). | -| `rlpdump` | Developer utility tool to convert binary RLP ([Recursive Length Prefix](https://eth.wiki/en/fundamentals/rlp)) dumps (data encoding used by the Ethereum protocol both network as well as consensus wise) to user-friendlier hierarchical representation (e.g. `rlpdump --hex CE0183FFFFFFC4C304050583616263`). | +| `rlpdump` | Developer utility tool to convert binary RLP ([Recursive Length Prefix](https://ethereum.org/en/developers/docs/data-structures-and-encoding/rlp)) dumps (data encoding used by the Ethereum protocol both network as well as consensus wise) to user-friendlier hierarchical representation (e.g. `rlpdump --hex CE0183FFFFFFC4C304050583616263`). | | `puppeth` | a CLI wizard that aids in creating a new Ethereum network. | ## Running `geth` @@ -65,14 +65,14 @@ Recommended: * Fast CPU with 4+ cores * 16GB+ RAM -* High Performance SSD with at least 1TB free space +* High-performance SSD with at least 1TB of free space * 25+ MBit/sec download Internet service ### Full node on the main Ethereum network By far the most common scenario is people wanting to simply interact with the Ethereum network: create accounts; transfer funds; deploy and interact with contracts. For this -particular use-case the user doesn't care about years-old historical data, so we can +particular use case, the user doesn't care about years-old historical data, so we can sync quickly to the current state of the network. To do so: ```shell @@ -83,11 +83,11 @@ This command will: * Start `geth` in snap sync mode (default, can be changed with the `--syncmode` flag), causing it to download more data in exchange for avoiding processing the entire history of the Ethereum network, which is very CPU intensive. - * Start up `geth`'s built-in interactive [JavaScript console](https://geth.ethereum.org/docs/interface/javascript-console), + * Start the built-in interactive [JavaScript console](https://geth.ethereum.org/docs/interface/javascript-console), (via the trailing `console` subcommand) through which you can interact using [`web3` methods](https://github.com/ChainSafe/web3.js/blob/0.20.7/DOCUMENTATION.md) (note: the `web3` version bundled within `geth` is very old, and not up to date with official docs), as well as `geth`'s own [management APIs](https://geth.ethereum.org/docs/rpc/server). - This tool is optional and if you leave it out you can always attach to an already running + This tool is optional and if you leave it out you can always attach it to an already running `geth` instance with `geth attach`. ### A Full node on the Görli test network @@ -102,12 +102,12 @@ the main network, but with play-Ether only. $ geth --goerli console ``` -The `console` subcommand has the exact same meaning as above and they are equally -useful on the testnet too. Please, see above for their explanations if you've skipped here. +The `console` subcommand has the same meaning as above and is equally +useful on the testnet too. Specifying the `--goerli` flag, however, will reconfigure your `geth` instance a bit: - * Instead of connecting the main Ethereum network, the client will connect to the Görli + * Instead of connecting to the main Ethereum network, the client will connect to the Görli test network, which uses different P2P bootnodes, different network IDs and genesis states. * Instead of using the default data directory (`~/.ethereum` on Linux for example), `geth` @@ -118,9 +118,9 @@ Specifying the `--goerli` flag, however, will reconfigure your `geth` instance a `geth attach /goerli/geth.ipc`. Windows users are not affected by this. -*Note: Although there are some internal protective measures to prevent transactions from -crossing over between the main network and test network, you should make sure to always -use separate accounts for play-money and real-money. Unless you manually move +*Note: Although some internal protective measures prevent transactions from +crossing over between the main network and test network, you should always +use separate accounts for play and real money. Unless you manually move accounts, `geth` will by default correctly separate the two networks and will not make any accounts available between them.* @@ -155,7 +155,7 @@ configuration file via: $ geth --config /path/to/your_config.toml ``` -To get an idea how the file should look like you can use the `dumpconfig` subcommand to +To get an idea of how the file should look like you can use the `dumpconfig` subcommand to export your existing configuration: ```shell @@ -175,7 +175,7 @@ docker run -d --name ethereum-node -v /Users/alice/ethereum:/root \ ethereum/client-go ``` -This will start `geth` in snap-sync mode with a DB memory allowance of 1GB just as the +This will start `geth` in snap-sync mode with a DB memory allowance of 1GB, as the above command does. It will also create a persistent volume in your home directory for saving your blockchain as well as map the default ports. There is also an `alpine` tag available for a slim version of the image. @@ -188,7 +188,7 @@ accessible from the outside. As a developer, sooner rather than later you'll want to start interacting with `geth` and the Ethereum network via your own programs and not manually through the console. To aid -this, `geth` has built-in support for a JSON-RPC based APIs ([standard APIs](https://eth.wiki/json-rpc/API) +this, `geth` has built-in support for a JSON-RPC based APIs ([standard APIs](https://ethereum.github.io/execution-apis/api-documentation/) and [`geth` specific APIs](https://geth.ethereum.org/docs/rpc/server)). These can be exposed via HTTP, WebSockets and IPC (UNIX sockets on UNIX based platforms, and named pipes on Windows). @@ -209,9 +209,9 @@ HTTP based JSON-RPC API options: * `--ws.addr` WS-RPC server listening interface (default: `localhost`) * `--ws.port` WS-RPC server listening port (default: `8546`) * `--ws.api` API's offered over the WS-RPC interface (default: `eth,net,web3`) - * `--ws.origins` Origins from which to accept websockets requests + * `--ws.origins` Origins from which to accept WebSocket requests * `--ipcdisable` Disable the IPC-RPC server - * `--ipcapi` API's offered over the IPC-RPC interface (default: `admin,debug,eth,miner,net,personal,shh,txpool,web3`) + * `--ipcapi` API's offered over the IPC-RPC interface (default: `admin,debug,eth,miner,net,personal,txpool,web3`) * `--ipcpath` Filename for IPC socket/pipe within the datadir (explicit paths escape it) You'll need to use your own programming environments' capabilities (libraries, tools, etc) to @@ -297,7 +297,7 @@ $ bootnode --genkey=boot.key $ bootnode --nodekey=boot.key ``` -With the bootnode online, it will display an [`enode` URL](https://eth.wiki/en/fundamentals/enode-url-format) +With the bootnode online, it will display an [`enode` URL](https://ethereum.org/en/developers/docs/networking-layer/network-addresses/#enode) that other nodes can use to connect to it and exchange peer information. Make sure to replace the displayed IP address information (most probably `[::]`) with your externally accessible IP to get the actual `enode` URL. @@ -327,7 +327,7 @@ requiring an OpenCL or CUDA enabled `ethminer` instance. For information on such setup, please consult the [EtherMining subreddit](https://www.reddit.com/r/EtherMining/) and the [ethminer](https://github.com/ethereum-mining/ethminer) repository. -In a private network setting, however a single CPU miner instance is more than enough for +In a private network setting, however, a single CPU miner instance is more than enough for practical purposes as it can produce a stable stream of blocks at the correct intervals without needing heavy resources (consider running on a single thread, no need for multiple ones either). To start a `geth` instance for mining, run it with all your usual flags, extended @@ -344,7 +344,7 @@ transactions are accepted at (`--miner.gasprice`). ## Contribution -Thank you for considering to help out with the source code! We welcome contributions +Thank you for considering helping out with the source code! We welcome contributions from anyone on the internet, and are grateful for even the smallest of fixes! If you'd like to contribute to go-ethereum, please fork, fix, commit and send a pull request @@ -374,6 +374,6 @@ The go-ethereum library (i.e. all code outside of the `cmd` directory) is licens [GNU Lesser General Public License v3.0](https://www.gnu.org/licenses/lgpl-3.0.en.html), also included in our repository in the `COPYING.LESSER` file. -The go-ethereum binaries (i.e. all code inside of the `cmd` directory) is licensed under the +The go-ethereum binaries (i.e. all code inside of the `cmd` directory) are licensed under the [GNU General Public License v3.0](https://www.gnu.org/licenses/gpl-3.0.en.html), also included in our repository in the `COPYING` file. diff --git a/accounts/abi/abi.go b/accounts/abi/abi.go index cd2f4d7978bd1..81bbee2f2b4a6 100644 --- a/accounts/abi/abi.go +++ b/accounts/abi/abi.go @@ -95,7 +95,7 @@ func (abi ABI) getArguments(name string, data []byte) (Arguments, error) { args = event.Inputs } if args == nil { - return nil, errors.New("abi: could not locate named method or event") + return nil, fmt.Errorf("abi: could not locate named method or event: %s", name) } return args, nil } @@ -164,7 +164,7 @@ func (abi *ABI) UnmarshalJSON(data []byte) error { case "constructor": abi.Constructor = NewMethod("", "", Constructor, field.StateMutability, field.Constant, field.Payable, field.Inputs, nil) case "function": - name := overloadedName(field.Name, func(s string) bool { _, ok := abi.Methods[s]; return ok }) + name := ResolveNameConflict(field.Name, func(s string) bool { _, ok := abi.Methods[s]; return ok }) abi.Methods[name] = NewMethod(name, field.Name, Function, field.StateMutability, field.Constant, field.Payable, field.Inputs, field.Outputs) case "fallback": // New introduced function type in v0.6.0, check more detail @@ -184,9 +184,11 @@ func (abi *ABI) UnmarshalJSON(data []byte) error { } abi.Receive = NewMethod("", "", Receive, field.StateMutability, field.Constant, field.Payable, nil, nil) case "event": - name := overloadedName(field.Name, func(s string) bool { _, ok := abi.Events[s]; return ok }) + name := ResolveNameConflict(field.Name, func(s string) bool { _, ok := abi.Events[s]; return ok }) abi.Events[name] = NewEvent(name, field.Name, field.Anonymous, field.Inputs) case "error": + // Errors cannot be overloaded or overridden but are inherited, + // no need to resolve the name conflict here. abi.Errors[field.Name] = NewError(field.Name, field.Inputs) default: return fmt.Errorf("abi: could not recognize type %v of field %v", field.Type, field.Name) @@ -251,20 +253,3 @@ func UnpackRevert(data []byte) (string, error) { } return unpacked[0].(string), nil } - -// overloadedName returns the next available name for a given thing. -// Needed since solidity allows for overloading. -// -// e.g. if the abi contains Methods send, send1 -// overloadedName would return send2 for input send. -// -// overloadedName works for methods, events and errors. -func overloadedName(rawName string, isAvail func(string) bool) string { - name := rawName - ok := isAvail(name) - for idx := 0; ok; idx++ { - name = fmt.Sprintf("%s%d", rawName, idx) - ok = isAvail(name) - } - return name -} diff --git a/accounts/abi/abi_test.go b/accounts/abi/abi_test.go index cc8dfc61c389f..96c11e096462a 100644 --- a/accounts/abi/abi_test.go +++ b/accounts/abi/abi_test.go @@ -165,8 +165,9 @@ func TestInvalidABI(t *testing.T) { // TestConstructor tests a constructor function. // The test is based on the following contract: -// contract TestConstructor { -// constructor(uint256 a, uint256 b) public{} +// +// contract TestConstructor { +// constructor(uint256 a, uint256 b) public{} // } func TestConstructor(t *testing.T) { json := `[{ "inputs": [{"internalType": "uint256","name": "a","type": "uint256" },{ "internalType": "uint256","name": "b","type": "uint256"}],"stateMutability": "nonpayable","type": "constructor"}]` @@ -724,16 +725,19 @@ func TestBareEvents(t *testing.T) { } // TestUnpackEvent is based on this contract: -// contract T { -// event received(address sender, uint amount, bytes memo); -// event receivedAddr(address sender); -// function receive(bytes memo) external payable { -// received(msg.sender, msg.value, memo); -// receivedAddr(msg.sender); -// } -// } +// +// contract T { +// event received(address sender, uint amount, bytes memo); +// event receivedAddr(address sender); +// function receive(bytes memo) external payable { +// received(msg.sender, msg.value, memo); +// receivedAddr(msg.sender); +// } +// } +// // When receive("X") is called with sender 0x00... and value 1, it produces this tx receipt: -// receipt{status=1 cgas=23949 bloom=00000000004000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000040200000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000080000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 logs=[log: b6818c8064f645cd82d99b59a1a267d6d61117ef [75fd880d39c1daf53b6547ab6cb59451fc6452d27caa90e5b6649dd8293b9eed] 000000000000000000000000376c47978271565f56deb45495afa69e59c16ab200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000060000000000000000000000000000000000000000000000000000000000000000158 9ae378b6d4409eada347a5dc0c180f186cb62dc68fcc0f043425eb917335aa28 0 95d429d309bb9d753954195fe2d69bd140b4ae731b9b5b605c34323de162cf00 0]} +// +// receipt{status=1 cgas=23949 bloom=00000000004000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000040200000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000080000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 logs=[log: b6818c8064f645cd82d99b59a1a267d6d61117ef [75fd880d39c1daf53b6547ab6cb59451fc6452d27caa90e5b6649dd8293b9eed] 000000000000000000000000376c47978271565f56deb45495afa69e59c16ab200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000060000000000000000000000000000000000000000000000000000000000000000158 9ae378b6d4409eada347a5dc0c180f186cb62dc68fcc0f043425eb917335aa28 0 95d429d309bb9d753954195fe2d69bd140b4ae731b9b5b605c34323de162cf00 0]} func TestUnpackEvent(t *testing.T) { const abiJSON = `[{"constant":false,"inputs":[{"name":"memo","type":"bytes"}],"name":"receive","outputs":[],"payable":true,"stateMutability":"payable","type":"function"},{"anonymous":false,"inputs":[{"indexed":false,"name":"sender","type":"address"},{"indexed":false,"name":"amount","type":"uint256"},{"indexed":false,"name":"memo","type":"bytes"}],"name":"received","type":"event"},{"anonymous":false,"inputs":[{"indexed":false,"name":"sender","type":"address"}],"name":"receivedAddr","type":"event"}]` abi, err := JSON(strings.NewReader(abiJSON)) @@ -1038,9 +1042,7 @@ func TestABI_EventById(t *testing.T) { } if event == nil { t.Errorf("We should find a event for topic %s, test #%d", topicID.Hex(), testnum) - } - - if event.ID != topicID { + } else if event.ID != topicID { t.Errorf("Event id %s does not match topic %s, test #%d", event.ID.Hex(), topicID.Hex(), testnum) } @@ -1080,8 +1082,9 @@ func TestDoubleDuplicateMethodNames(t *testing.T) { // TestDoubleDuplicateEventNames checks that if send0 already exists, there won't be a name // conflict and that the second send event will be renamed send1. // The test runs the abi of the following contract. -// contract DuplicateEvent { -// event send(uint256 a); +// +// contract DuplicateEvent { +// event send(uint256 a); // event send0(); // event send(); // } @@ -1108,7 +1111,8 @@ func TestDoubleDuplicateEventNames(t *testing.T) { // TestUnnamedEventParam checks that an event with unnamed parameters is // correctly handled. // The test runs the abi of the following contract. -// contract TestEvent { +// +// contract TestEvent { // event send(uint256, uint256); // } func TestUnnamedEventParam(t *testing.T) { diff --git a/accounts/abi/argument.go b/accounts/abi/argument.go index c5326d5700a65..2e48d539e0dc2 100644 --- a/accounts/abi/argument.go +++ b/accounts/abi/argument.go @@ -18,6 +18,7 @@ package abi import ( "encoding/json" + "errors" "fmt" "reflect" "strings" @@ -79,7 +80,7 @@ func (arguments Arguments) isTuple() bool { func (arguments Arguments) Unpack(data []byte) ([]interface{}, error) { if len(data) == 0 { if len(arguments.NonIndexed()) != 0 { - return nil, fmt.Errorf("abi: attempting to unmarshall an empty string while arguments are expected") + return nil, errors.New("abi: attempting to unmarshall an empty string while arguments are expected") } return make([]interface{}, 0), nil } @@ -90,11 +91,11 @@ func (arguments Arguments) Unpack(data []byte) ([]interface{}, error) { func (arguments Arguments) UnpackIntoMap(v map[string]interface{}, data []byte) error { // Make sure map is not nil if v == nil { - return fmt.Errorf("abi: cannot unpack into a nil map") + return errors.New("abi: cannot unpack into a nil map") } if len(data) == 0 { if len(arguments.NonIndexed()) != 0 { - return fmt.Errorf("abi: attempting to unmarshall an empty string while arguments are expected") + return errors.New("abi: attempting to unmarshall an empty string while arguments are expected") } return nil // Nothing to unmarshal, return } @@ -116,7 +117,7 @@ func (arguments Arguments) Copy(v interface{}, values []interface{}) error { } if len(values) == 0 { if len(arguments.NonIndexed()) != 0 { - return fmt.Errorf("abi: attempting to copy no values while arguments are expected") + return errors.New("abi: attempting to copy no values while arguments are expected") } return nil // Nothing to copy, return } @@ -186,6 +187,9 @@ func (arguments Arguments) UnpackValues(data []byte) ([]interface{}, error) { virtualArgs := 0 for index, arg := range nonIndexedArgs { marshalledValue, err := toGoType((index+virtualArgs)*32, arg.Type, data) + if err != nil { + return nil, err + } if arg.Type.T == ArrayTy && !isDynamicType(arg.Type) { // If we have a static array, like [3]uint256, these are coded as // just like uint256,uint256,uint256. @@ -203,9 +207,6 @@ func (arguments Arguments) UnpackValues(data []byte) ([]interface{}, error) { // coded as just like uint256,bool,uint256 virtualArgs += getTypeSize(arg.Type)/32 - 1 } - if err != nil { - return nil, err - } retval = append(retval, marshalledValue) } return retval, nil diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go index 48415802b36f6..3ddcda794f2ae 100644 --- a/accounts/abi/bind/backends/simulated.go +++ b/accounts/abi/bind/backends/simulated.go @@ -67,8 +67,10 @@ type SimulatedBackend struct { stuckTransactions types.Transactions // holds onto all txes that don't go into the pending block due to low gas pendingBlock *types.Block // Currently pending block that will be imported on request pendingState *state.StateDB // Currently pending state that will be the active on request + pendingReceipts types.Receipts // Currently receipts for the pending block - events *filters.EventSystem // Event system for filtering log events live + events *filters.EventSystem // for filtering log events live + filterSystem *filters.FilterSystem // for filtering database logs config *params.ChainConfig @@ -79,16 +81,23 @@ type SimulatedBackend struct { // and uses a simulated blockchain for testing purposes. // A simulated backend always uses chainID 1337. func NewSimulatedBackendWithDatabase(database ethdb.Database, alloc core.GenesisAlloc, gasLimit uint64) *SimulatedBackend { - genesis := core.Genesis{Config: params.AllEthashProtocolChanges, GasLimit: gasLimit, Alloc: alloc} - genesis.MustCommit(database) - blockchain, _ := core.NewBlockChain(database, nil, genesis.Config, ethash.NewFaker(), vm.Config{}, nil, nil) + genesis := core.Genesis{ + Config: params.AllEthashProtocolChanges, + GasLimit: gasLimit, + Alloc: alloc, + } + blockchain, _ := core.NewBlockChain(database, nil, &genesis, nil, ethash.NewFaker(), vm.Config{}, nil, nil) backend := &SimulatedBackend{ database: database, blockchain: blockchain, config: genesis.Config, - events: filters.NewEventSystem(&filterBackend{database, blockchain}, false), } + + filterBackend := &filterBackend{database, blockchain, backend} + backend.filterSystem = filters.NewFilterSystem(filterBackend, filters.Config{}) + backend.events = filters.NewEventSystem(backend.filterSystem, false) + backend.rollback(blockchain.CurrentBlock()) return backend } @@ -108,7 +117,7 @@ func (b *SimulatedBackend) Close() error { // Commit imports all the pending transactions as a single block and starts a // fresh new state. -func (b *SimulatedBackend) Commit() { +func (b *SimulatedBackend) Commit() common.Hash { b.mu.Lock() defer b.mu.Unlock() @@ -130,10 +139,14 @@ func (b *SimulatedBackend) Commit() { if _, err := b.blockchain.InsertChain([]*types.Block{blocks[0]}); err != nil { panic(err) // This cannot happen unless the simulator is wrong, fail in that case } + blockHash := b.pendingBlock.Hash() + // Using the last inserted block here makes it possible to build on a side // chain after a fork. b.stuckTransactions = remainingTx b.rollback(blocks[0]) + + return blockHash } // Rollback aborts all pending transactions, reverting to the last committed state. @@ -631,7 +644,7 @@ func (b *SimulatedBackend) callContract(ctx context.Context, call ethereum.CallM // User specified the legacy gas field, convert to 1559 gas typing call.GasFeeCap, call.GasTipCap = call.GasPrice, call.GasPrice } else { - // User specified 1559 gas feilds (or none), use those + // User specified 1559 gas fields (or none), use those if call.GasFeeCap == nil { call.GasFeeCap = new(big.Int) } @@ -690,7 +703,7 @@ func (b *SimulatedBackend) SendTransaction(ctx context.Context, tx *types.Transa } var remainingTxes types.Transactions // Include tx in chain - blocks, _ := core.GenerateChain(b.config, block, ethash.NewFaker(), b.database, 1, func(number int, block *core.BlockGen) { + blocks, receipts := core.GenerateChain(b.config, block, ethash.NewFaker(), b.database, 1, func(number int, block *core.BlockGen) { for _, tx := range b.pendingBlock.Transactions() { // if market gas price is not set or tx gas price is lower than the set market gas price if b.marketGasPrice == nil || b.marketGasPrice.Cmp(tx.GasPrice()) <= 0 { @@ -712,6 +725,9 @@ func (b *SimulatedBackend) SendTransaction(ctx context.Context, tx *types.Transa b.pendingState, _ = state.New(b.pendingBlock.Root(), stateDB.Database(), nil) b.stuckTransactions = remainingTxes + b.pendingReceipts = receipts[0] + b.stuckTransactions = remainingTxes + return nil } @@ -723,7 +739,7 @@ func (b *SimulatedBackend) FilterLogs(ctx context.Context, query ethereum.Filter var filter *filters.Filter if query.BlockHash != nil { // Block filter requested, construct a single-shot filter - filter = filters.NewBlockFilter(&filterBackend{b.database, b.blockchain}, *query.BlockHash, query.Addresses, query.Topics) + filter = b.filterSystem.NewBlockFilter(*query.BlockHash, query.Addresses, query.Topics) } else { // Initialize unset filter boundaries to run from genesis to chain head from := int64(0) @@ -735,7 +751,7 @@ func (b *SimulatedBackend) FilterLogs(ctx context.Context, query ethereum.Filter to = query.ToBlock.Int64() } // Construct the range filter - filter = filters.NewRangeFilter(&filterBackend{b.database, b.blockchain}, from, to, query.Addresses, query.Topics) + filter = b.filterSystem.NewRangeFilter(from, to, query.Addresses, query.Topics) } // Run the filter and return all the logs logs, err := filter.Logs(ctx) @@ -819,8 +835,13 @@ func (b *SimulatedBackend) AdjustTime(adjustment time.Duration) error { if len(b.pendingBlock.Transactions()) != 0 { return errors.New("Could not adjust time on non-empty block") } + // Get the last block + block := b.blockchain.GetBlockByHash(b.pendingBlock.ParentHash()) + if block == nil { + return fmt.Errorf("could not find parent") + } - blocks, _ := core.GenerateChain(b.config, b.blockchain.CurrentBlock(), ethash.NewFaker(), b.database, 1, func(number int, block *core.BlockGen) { + blocks, _ := core.GenerateChain(b.config, block, ethash.NewFaker(), b.database, 1, func(number int, block *core.BlockGen) { block.OffsetTime(int64(adjustment.Seconds())) }) stateDB, _ := b.blockchain.State() @@ -856,11 +877,13 @@ func (m callMsg) AccessList() types.AccessList { return m.CallMsg.AccessList } // filterBackend implements filters.Backend to support filtering for logs without // taking bloom-bits acceleration structures into account. type filterBackend struct { - db ethdb.Database - bc *core.BlockChain + db ethdb.Database + bc *core.BlockChain + backend *SimulatedBackend } -func (fb *filterBackend) ChainDb() ethdb.Database { return fb.db } +func (fb *filterBackend) ChainDb() ethdb.Database { return fb.db } + func (fb *filterBackend) EventMux() *event.TypeMux { panic("not supported") } func (fb *filterBackend) HeaderByNumber(ctx context.Context, block rpc.BlockNumber) (*types.Header, error) { @@ -874,6 +897,10 @@ func (fb *filterBackend) HeaderByHash(ctx context.Context, hash common.Hash) (*t return fb.bc.GetHeaderByHash(hash), nil } +func (fb *filterBackend) PendingBlockAndReceipts() (*types.Block, types.Receipts) { + return fb.backend.pendingBlock, fb.backend.pendingReceipts +} + func (fb *filterBackend) GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) { number := rawdb.ReadHeaderNumber(fb.db, hash) if number == nil { @@ -882,19 +909,8 @@ func (fb *filterBackend) GetReceipts(ctx context.Context, hash common.Hash) (typ return rawdb.ReadReceipts(fb.db, hash, *number, fb.bc.Config()), nil } -func (fb *filterBackend) GetLogs(ctx context.Context, hash common.Hash) ([][]*types.Log, error) { - number := rawdb.ReadHeaderNumber(fb.db, hash) - if number == nil { - return nil, nil - } - receipts := rawdb.ReadReceipts(fb.db, hash, *number, fb.bc.Config()) - if receipts == nil { - return nil, nil - } - logs := make([][]*types.Log, len(receipts)) - for i, receipt := range receipts { - logs[i] = receipt.Logs - } +func (fb *filterBackend) GetLogs(ctx context.Context, hash common.Hash, number uint64) ([][]*types.Log, error) { + logs := rawdb.ReadLogs(fb.db, hash, number, fb.bc.Config()) return logs, nil } diff --git a/accounts/abi/bind/backends/simulated_test.go b/accounts/abi/bind/backends/simulated_test.go index 4ec74a5e598b8..91e3c7a7f7c4d 100644 --- a/accounts/abi/bind/backends/simulated_test.go +++ b/accounts/abi/bind/backends/simulated_test.go @@ -96,17 +96,18 @@ func TestSimulatedBackend(t *testing.T) { var testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") -// the following is based on this contract: -// contract T { -// event received(address sender, uint amount, bytes memo); -// event receivedAddr(address sender); +// the following is based on this contract: // -// function receive(bytes calldata memo) external payable returns (string memory res) { -// emit received(msg.sender, msg.value, memo); -// emit receivedAddr(msg.sender); -// return "hello world"; -// } -// } +// contract T { +// event received(address sender, uint amount, bytes memo); +// event receivedAddr(address sender); +// +// function receive(bytes calldata memo) external payable returns (string memory res) { +// emit received(msg.sender, msg.value, memo); +// emit receivedAddr(msg.sender); +// return "hello world"; +// } +// } const abiJSON = `[ { "constant": false, "inputs": [ { "name": "memo", "type": "bytes" } ], "name": "receive", "outputs": [ { "name": "res", "type": "string" } ], "payable": true, "stateMutability": "payable", "type": "function" }, { "anonymous": false, "inputs": [ { "indexed": false, "name": "sender", "type": "address" }, { "indexed": false, "name": "amount", "type": "uint256" }, { "indexed": false, "name": "memo", "type": "bytes" } ], "name": "received", "type": "event" }, { "anonymous": false, "inputs": [ { "indexed": false, "name": "sender", "type": "address" } ], "name": "receivedAddr", "type": "event" } ]` const abiBin = `0x608060405234801561001057600080fd5b506102a0806100206000396000f3fe60806040526004361061003b576000357c010000000000000000000000000000000000000000000000000000000090048063a69b6ed014610040575b600080fd5b6100b76004803603602081101561005657600080fd5b810190808035906020019064010000000081111561007357600080fd5b82018360208201111561008557600080fd5b803590602001918460018302840111640100000000831117156100a757600080fd5b9091929391929390505050610132565b6040518080602001828103825283818151815260200191508051906020019080838360005b838110156100f75780820151818401526020810190506100dc565b50505050905090810190601f1680156101245780820380516001836020036101000a031916815260200191505b509250505060405180910390f35b60607f75fd880d39c1daf53b6547ab6cb59451fc6452d27caa90e5b6649dd8293b9eed33348585604051808573ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff168152602001848152602001806020018281038252848482818152602001925080828437600081840152601f19601f8201169050808301925050509550505050505060405180910390a17f46923992397eac56cf13058aced2a1871933622717e27b24eabc13bf9dd329c833604051808273ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16815260200191505060405180910390a16040805190810160405280600b81526020017f68656c6c6f20776f726c6400000000000000000000000000000000000000000081525090509291505056fea165627a7a72305820ff0c57dad254cfeda48c9cfb47f1353a558bccb4d1bc31da1dae69315772d29e0029` const deployedCode = `60806040526004361061003b576000357c010000000000000000000000000000000000000000000000000000000090048063a69b6ed014610040575b600080fd5b6100b76004803603602081101561005657600080fd5b810190808035906020019064010000000081111561007357600080fd5b82018360208201111561008557600080fd5b803590602001918460018302840111640100000000831117156100a757600080fd5b9091929391929390505050610132565b6040518080602001828103825283818151815260200191508051906020019080838360005b838110156100f75780820151818401526020810190506100dc565b50505050905090810190601f1680156101245780820380516001836020036101000a031916815260200191505b509250505060405180910390f35b60607f75fd880d39c1daf53b6547ab6cb59451fc6452d27caa90e5b6649dd8293b9eed33348585604051808573ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff168152602001848152602001806020018281038252848482818152602001925080828437600081840152601f19601f8201169050808301925050509550505050505060405180910390a17f46923992397eac56cf13058aced2a1871933622717e27b24eabc13bf9dd329c833604051808273ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16815260200191505060405180910390a16040805190810160405280600b81526020017f68656c6c6f20776f726c6400000000000000000000000000000000000000000081525090509291505056fea165627a7a72305820ff0c57dad254cfeda48c9cfb47f1353a558bccb4d1bc31da1dae69315772d29e0029` @@ -420,12 +421,13 @@ func TestEstimateGas(t *testing.T) { /* pragma solidity ^0.6.4; contract GasEstimation { - function PureRevert() public { revert(); } - function Revert() public { revert("revert reason");} - function OOG() public { for (uint i = 0; ; i++) {}} - function Assert() public { assert(false);} - function Valid() public {} - }*/ + function PureRevert() public { revert(); } + function Revert() public { revert("revert reason");} + function OOG() public { for (uint i = 0; ; i++) {}} + function Assert() public { assert(false);} + function Valid() public {} + } + */ const contractAbi = "[{\"inputs\":[],\"name\":\"Assert\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"OOG\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"PureRevert\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"Revert\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"Valid\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"}]" const contractBin = "0x60806040523480156100115760006000fd5b50610017565b61016e806100266000396000f3fe60806040523480156100115760006000fd5b506004361061005c5760003560e01c806350f6fe3414610062578063aa8b1d301461006c578063b9b046f914610076578063d8b9839114610080578063e09fface1461008a5761005c565b60006000fd5b61006a610094565b005b6100746100ad565b005b61007e6100b5565b005b6100886100c2565b005b610092610135565b005b6000600090505b5b808060010191505061009b565b505b565b60006000fd5b565b600015156100bf57fe5b5b565b6040517f08c379a000000000000000000000000000000000000000000000000000000000815260040180806020018281038252600d8152602001807f72657665727420726561736f6e0000000000000000000000000000000000000081526020015060200191505060405180910390fd5b565b5b56fea2646970667358221220345bbcbb1a5ecf22b53a78eaebf95f8ee0eceff6d10d4b9643495084d2ec934a64736f6c63430006040033" @@ -658,8 +660,7 @@ func TestHeaderByNumber(t *testing.T) { } if latestBlockHeader == nil { t.Errorf("received a nil block header") - } - if latestBlockHeader.Number.Uint64() != uint64(0) { + } else if latestBlockHeader.Number.Uint64() != uint64(0) { t.Errorf("expected block header number 0, instead got %v", latestBlockHeader.Number.Uint64()) } @@ -998,7 +999,8 @@ func TestCodeAt(t *testing.T) { } // When receive("X") is called with sender 0x00... and value 1, it produces this tx receipt: -// receipt{status=1 cgas=23949 bloom=00000000004000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000040200000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000080000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 logs=[log: b6818c8064f645cd82d99b59a1a267d6d61117ef [75fd880d39c1daf53b6547ab6cb59451fc6452d27caa90e5b6649dd8293b9eed] 000000000000000000000000376c47978271565f56deb45495afa69e59c16ab200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000060000000000000000000000000000000000000000000000000000000000000000158 9ae378b6d4409eada347a5dc0c180f186cb62dc68fcc0f043425eb917335aa28 0 95d429d309bb9d753954195fe2d69bd140b4ae731b9b5b605c34323de162cf00 0]} +// +// receipt{status=1 cgas=23949 bloom=00000000004000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000040200000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000080000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 logs=[log: b6818c8064f645cd82d99b59a1a267d6d61117ef [75fd880d39c1daf53b6547ab6cb59451fc6452d27caa90e5b6649dd8293b9eed] 000000000000000000000000376c47978271565f56deb45495afa69e59c16ab200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000060000000000000000000000000000000000000000000000000000000000000000158 9ae378b6d4409eada347a5dc0c180f186cb62dc68fcc0f043425eb917335aa28 0 95d429d309bb9d753954195fe2d69bd140b4ae731b9b5b605c34323de162cf00 0]} func TestPendingAndCallContract(t *testing.T) { testAddr := crypto.PubkeyToAddress(testKey.PublicKey) sim := simTestBackend(testAddr) @@ -1061,27 +1063,27 @@ func TestPendingAndCallContract(t *testing.T) { // This test is based on the following contract: /* contract Reverter { - function revertString() public pure{ - require(false, "some error"); - } - function revertNoString() public pure { - require(false, ""); - } - function revertASM() public pure { - assembly { - revert(0x0, 0x0) - } - } - function noRevert() public pure { - assembly { - // Assembles something that looks like require(false, "some error") but is not reverted - mstore(0x0, 0x08c379a000000000000000000000000000000000000000000000000000000000) - mstore(0x4, 0x0000000000000000000000000000000000000000000000000000000000000020) - mstore(0x24, 0x000000000000000000000000000000000000000000000000000000000000000a) - mstore(0x44, 0x736f6d65206572726f7200000000000000000000000000000000000000000000) - return(0x0, 0x64) - } - } + function revertString() public pure{ + require(false, "some error"); + } + function revertNoString() public pure { + require(false, ""); + } + function revertASM() public pure { + assembly { + revert(0x0, 0x0) + } + } + function noRevert() public pure { + assembly { + // Assembles something that looks like require(false, "some error") but is not reverted + mstore(0x0, 0x08c379a000000000000000000000000000000000000000000000000000000000) + mstore(0x4, 0x0000000000000000000000000000000000000000000000000000000000000020) + mstore(0x24, 0x000000000000000000000000000000000000000000000000000000000000000a) + mstore(0x44, 0x736f6d65206572726f7200000000000000000000000000000000000000000000) + return(0x0, 0x64) + } + } }*/ func TestCallContractRevert(t *testing.T) { testAddr := crypto.PubkeyToAddress(testKey.PublicKey) @@ -1208,11 +1210,11 @@ func TestFork(t *testing.T) { /* Example contract to test event emission: -pragma solidity >=0.7.0 <0.9.0; -contract Callable { - event Called(); - function Call() public { emit Called(); } -} + pragma solidity >=0.7.0 <0.9.0; + contract Callable { + event Called(); + function Call() public { emit Called(); } + } */ const callableAbi = "[{\"anonymous\":false,\"inputs\":[],\"name\":\"Called\",\"type\":\"event\"},{\"inputs\":[],\"name\":\"Call\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"}]" @@ -1230,7 +1232,7 @@ const callableBin = "6080604052348015600f57600080fd5b5060998061001e6000396000f3f // 7. Mine two blocks to trigger a reorg. // 8. Check that the event was removed. // 9. Re-send the transaction and mine a block. -// 10. Check that the event was reborn. +// 10. Check that the event was reborn. func TestForkLogsReborn(t *testing.T) { testAddr := crypto.PubkeyToAddress(testKey.PublicKey) sim := simTestBackend(testAddr) @@ -1355,15 +1357,15 @@ func TestLowGasTxNotBeingMined(t *testing.T) { defer sim.Close() var testCases = []struct { - name string - txType byte + name string + txType byte }{ { - name: "LegacyTx", + name: "LegacyTx", txType: types.LegacyTxType, }, { - name: "dynamicTx", + name: "dynamicTx", txType: types.DynamicFeeTxType, }, } @@ -1414,7 +1416,7 @@ func TestLowGasTxNotBeingMined(t *testing.T) { // expect nonce be the same because low gas fee tx will not be mined nonce, err := sim.PendingNonceAt(bgCtx, testAddr) require.NoError(t, err) - assert.Equal(t, uint64(i), nonce) + assert.Equal(t, uint64(i), nonce) // send tx with higher gas fee sufficientGasFeeTx := types.NewTx(&types.LegacyTx{ @@ -1440,7 +1442,7 @@ func TestLowGasTxNotBeingMined(t *testing.T) { // expect nonce has increased nonce, err = sim.PendingNonceAt(bgCtx, testAddr) require.NoError(t, err) - assert.Equal(t, uint64(i) + 1, nonce) + assert.Equal(t, uint64(i)+1, nonce) }) } } @@ -1454,15 +1456,15 @@ func TestLowGasTxNotBeingMined(t *testing.T) { // 4. Check tx get mined func TestLowGasTxGetMinedOnceGasFeeDropped(t *testing.T) { var testCases = []struct { - name string - txType byte + name string + txType byte }{ { - name: "LegacyTx", + name: "LegacyTx", txType: types.LegacyTxType, }, { - name: "dynamicTx", + name: "dynamicTx", txType: types.DynamicFeeTxType, }, } @@ -1528,7 +1530,7 @@ func TestLowGasTxGetMinedOnceGasFeeDropped(t *testing.T) { // nonce has increased pendingNonce, err = sim.PendingNonceAt(bgCtx, testAddr) require.NoError(t, err) - assert.Equal(t, uint64(i) + 1, pendingNonce) + assert.Equal(t, uint64(i)+1, pendingNonce) // the transaction should have been mined _, isPending, err := sim.TransactionByHash(bgCtx, signedTx.Hash()) @@ -1537,3 +1539,61 @@ func TestLowGasTxGetMinedOnceGasFeeDropped(t *testing.T) { }) } } +func TestCommitReturnValue(t *testing.T) { + testAddr := crypto.PubkeyToAddress(testKey.PublicKey) + sim := simTestBackend(testAddr) + defer sim.Close() + + startBlockHeight := sim.blockchain.CurrentBlock().NumberU64() + + // Test if Commit returns the correct block hash + h1 := sim.Commit() + if h1 != sim.blockchain.CurrentBlock().Hash() { + t.Error("Commit did not return the hash of the last block.") + } + + // Create a block in the original chain (containing a transaction to force different block hashes) + head, _ := sim.HeaderByNumber(context.Background(), nil) // Should be child's, good enough + gasPrice := new(big.Int).Add(head.BaseFee, big.NewInt(1)) + _tx := types.NewTransaction(0, testAddr, big.NewInt(1000), params.TxGas, gasPrice, nil) + tx, _ := types.SignTx(_tx, types.HomesteadSigner{}, testKey) + sim.SendTransaction(context.Background(), tx) + h2 := sim.Commit() + + // Create another block in the original chain + sim.Commit() + + // Fork at the first bock + if err := sim.Fork(context.Background(), h1); err != nil { + t.Errorf("forking: %v", err) + } + + // Test if Commit returns the correct block hash after the reorg + h2fork := sim.Commit() + if h2 == h2fork { + t.Error("The block in the fork and the original block are the same block!") + } + if sim.blockchain.GetHeader(h2fork, startBlockHeight+2) == nil { + t.Error("Could not retrieve the just created block (side-chain)") + } +} + +// TestAdjustTimeAfterFork ensures that after a fork, AdjustTime uses the pending fork +// block's parent rather than the canonical head's parent. +func TestAdjustTimeAfterFork(t *testing.T) { + testAddr := crypto.PubkeyToAddress(testKey.PublicKey) + sim := simTestBackend(testAddr) + defer sim.Close() + + sim.Commit() // h1 + h1 := sim.blockchain.CurrentHeader().Hash() + sim.Commit() // h2 + sim.Fork(context.Background(), h1) + sim.AdjustTime(1 * time.Second) + sim.Commit() + + head := sim.blockchain.CurrentHeader() + if head.Number == common.Big2 && head.ParentHash != h1 { + t.Errorf("failed to build block on fork") + } +} diff --git a/accounts/abi/bind/base.go b/accounts/abi/bind/base.go index fe330014d35a2..88b997684a40c 100644 --- a/accounts/abi/bind/base.go +++ b/accounts/abi/bind/base.go @@ -32,6 +32,8 @@ import ( "github.com/ethereum/go-ethereum/event" ) +const basefeeWiggleMultiplier = 2 + // SignerFn is a signer function callback when a contract requires a method to // sign the transaction before submission. type SignerFn func(common.Address, *types.Transaction) (*types.Transaction, error) @@ -254,7 +256,7 @@ func (c *BoundContract) createDynamicTx(opts *TransactOpts, contract *common.Add if gasFeeCap == nil { gasFeeCap = new(big.Int).Add( gasTipCap, - new(big.Int).Mul(head.BaseFee, big.NewInt(2)), + new(big.Int).Mul(head.BaseFee, big.NewInt(basefeeWiggleMultiplier)), ) } if gasFeeCap.Cmp(gasTipCap) < 0 { diff --git a/accounts/abi/bind/base_test.go b/accounts/abi/bind/base_test.go index 25b2f8a865f26..2307b9874b18f 100644 --- a/accounts/abi/bind/base_test.go +++ b/accounts/abi/bind/base_test.go @@ -115,7 +115,6 @@ func (mc *mockPendingCaller) PendingCallContract(ctx context.Context, call ether } func TestPassingBlockNumber(t *testing.T) { - mc := &mockPendingCaller{ mockCaller: &mockCaller{ codeAtBytes: []byte{1, 2, 3}, diff --git a/accounts/abi/bind/bind.go b/accounts/abi/bind/bind.go index 2bd8b6dde07f0..dac43f70e234b 100644 --- a/accounts/abi/bind/bind.go +++ b/accounts/abi/bind/bind.go @@ -43,6 +43,43 @@ const ( LangObjC ) +func isKeyWord(arg string) bool { + switch arg { + case "break": + case "case": + case "chan": + case "const": + case "continue": + case "default": + case "defer": + case "else": + case "fallthrough": + case "for": + case "func": + case "go": + case "goto": + case "if": + case "import": + case "interface": + case "iota": + case "map": + case "make": + case "new": + case "package": + case "range": + case "return": + case "select": + case "struct": + case "switch": + case "type": + case "var": + default: + return false + } + + return true +} + // Bind generates a Go wrapper around a contract ABI. This wrapper isn't meant // to be used as is in client code, but rather as an intermediate struct which // enforces compile time type safety and naming convention opposed to having to @@ -99,6 +136,7 @@ func Bind(types []string, abis []string, bytecodes []string, fsigs []map[string] // Normalize the method for capital cases and non-anonymous inputs/outputs normalized := original normalizedName := methodNormalizer[lang](alias(aliases, original.Name)) + // Ensure there is no duplicated identifier var identifiers = callIdentifiers if !original.IsConstant() { @@ -108,11 +146,12 @@ func Bind(types []string, abis []string, bytecodes []string, fsigs []map[string] return "", fmt.Errorf("duplicated identifier \"%s\"(normalized \"%s\"), use --alias for renaming", original.Name, normalizedName) } identifiers[normalizedName] = true + normalized.Name = normalizedName normalized.Inputs = make([]abi.Argument, len(original.Inputs)) copy(normalized.Inputs, original.Inputs) for j, input := range normalized.Inputs { - if input.Name == "" { + if input.Name == "" || isKeyWord(input.Name) { normalized.Inputs[j].Name = fmt.Sprintf("arg%d", j) } if hasStruct(input.Type) { @@ -152,12 +191,22 @@ func Bind(types []string, abis []string, bytecodes []string, fsigs []map[string] eventIdentifiers[normalizedName] = true normalized.Name = normalizedName + used := make(map[string]bool) normalized.Inputs = make([]abi.Argument, len(original.Inputs)) copy(normalized.Inputs, original.Inputs) for j, input := range normalized.Inputs { - if input.Name == "" { + if input.Name == "" || isKeyWord(input.Name) { normalized.Inputs[j].Name = fmt.Sprintf("arg%d", j) } + // Event is a bit special, we need to define event struct in binding, + // ensure there is no camel-case-style name conflict. + for index := 0; ; index++ { + if !used[capitalise(normalized.Inputs[j].Name)] { + used[capitalise(normalized.Inputs[j].Name)] = true + break + } + normalized.Inputs[j].Name = fmt.Sprintf("%s%d", normalized.Inputs[j].Name, index) + } if hasStruct(input.Type) { bindStructType[lang](input.Type, structs) } @@ -432,15 +481,22 @@ func bindStructTypeGo(kind abi.Type, structs map[string]*tmplStruct) string { if s, exist := structs[id]; exist { return s.Name } - var fields []*tmplField + var ( + names = make(map[string]bool) + fields []*tmplField + ) for i, elem := range kind.TupleElems { - field := bindStructTypeGo(*elem, structs) - fields = append(fields, &tmplField{Type: field, Name: capitalise(kind.TupleRawNames[i]), SolKind: *elem}) + name := capitalise(kind.TupleRawNames[i]) + name = abi.ResolveNameConflict(name, func(s string) bool { return names[s] }) + names[name] = true + fields = append(fields, &tmplField{Type: bindStructTypeGo(*elem, structs), Name: name, SolKind: *elem}) } name := kind.TupleRawName if name == "" { name = fmt.Sprintf("Struct%d", len(structs)) } + name = capitalise(name) + structs[id] = &tmplStruct{ Name: name, Fields: fields, diff --git a/accounts/abi/bind/bind_test.go b/accounts/abi/bind/bind_test.go index 87e8187f09ca0..5fa803849df6f 100644 --- a/accounts/abi/bind/bind_test.go +++ b/accounts/abi/bind/bind_test.go @@ -1954,6 +1954,91 @@ var bindTests = []struct { } `, }, + { + name: `NameConflict`, + contract: ` + // SPDX-License-Identifier: GPL-3.0 + pragma solidity >=0.4.22 <0.9.0; + contract oracle { + struct request { + bytes data; + bytes _data; + } + event log (int msg, int _msg); + function addRequest(request memory req) public pure {} + function getRequest() pure public returns (request memory) { + return request("", ""); + } + } + `, + bytecode: []string{"0x608060405234801561001057600080fd5b5061042b806100206000396000f3fe608060405234801561001057600080fd5b50600436106100365760003560e01c8063c2bb515f1461003b578063cce7b04814610059575b600080fd5b610043610075565b60405161005091906101af565b60405180910390f35b610073600480360381019061006e91906103ac565b6100b5565b005b61007d6100b8565b604051806040016040528060405180602001604052806000815250815260200160405180602001604052806000815250815250905090565b50565b604051806040016040528060608152602001606081525090565b600081519050919050565b600082825260208201905092915050565b60005b8381101561010c5780820151818401526020810190506100f1565b8381111561011b576000848401525b50505050565b6000601f19601f8301169050919050565b600061013d826100d2565b61014781856100dd565b93506101578185602086016100ee565b61016081610121565b840191505092915050565b600060408301600083015184820360008601526101888282610132565b915050602083015184820360208601526101a28282610132565b9150508091505092915050565b600060208201905081810360008301526101c9818461016b565b905092915050565b6000604051905090565b600080fd5b600080fd5b600080fd5b7f4e487b7100000000000000000000000000000000000000000000000000000000600052604160045260246000fd5b61022282610121565b810181811067ffffffffffffffff82111715610241576102406101ea565b5b80604052505050565b60006102546101d1565b90506102608282610219565b919050565b600080fd5b600080fd5b600080fd5b600067ffffffffffffffff82111561028f5761028e6101ea565b5b61029882610121565b9050602081019050919050565b82818337600083830152505050565b60006102c76102c284610274565b61024a565b9050828152602081018484840111156102e3576102e261026f565b5b6102ee8482856102a5565b509392505050565b600082601f83011261030b5761030a61026a565b5b813561031b8482602086016102b4565b91505092915050565b60006040828403121561033a576103396101e5565b5b610344604061024a565b9050600082013567ffffffffffffffff81111561036457610363610265565b5b610370848285016102f6565b600083015250602082013567ffffffffffffffff81111561039457610393610265565b5b6103a0848285016102f6565b60208301525092915050565b6000602082840312156103c2576103c16101db565b5b600082013567ffffffffffffffff8111156103e0576103df6101e0565b5b6103ec84828501610324565b9150509291505056fea264697066735822122033bca1606af9b6aeba1673f98c52003cec19338539fb44b86690ce82c51483b564736f6c634300080e0033"}, + abi: []string{`[ { "anonymous": false, "inputs": [ { "indexed": false, "internalType": "int256", "name": "msg", "type": "int256" }, { "indexed": false, "internalType": "int256", "name": "_msg", "type": "int256" } ], "name": "log", "type": "event" }, { "inputs": [ { "components": [ { "internalType": "bytes", "name": "data", "type": "bytes" }, { "internalType": "bytes", "name": "_data", "type": "bytes" } ], "internalType": "struct oracle.request", "name": "req", "type": "tuple" } ], "name": "addRequest", "outputs": [], "stateMutability": "pure", "type": "function" }, { "inputs": [], "name": "getRequest", "outputs": [ { "components": [ { "internalType": "bytes", "name": "data", "type": "bytes" }, { "internalType": "bytes", "name": "_data", "type": "bytes" } ], "internalType": "struct oracle.request", "name": "", "type": "tuple" } ], "stateMutability": "pure", "type": "function" } ]`}, + imports: ` + "math/big" + + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/eth/ethconfig" + `, + tester: ` + var ( + key, _ = crypto.GenerateKey() + user, _ = bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) + sim = backends.NewSimulatedBackend(core.GenesisAlloc{user.From: {Balance: big.NewInt(1000000000000000000)}}, ethconfig.Defaults.Miner.GasCeil) + ) + defer sim.Close() + + _, tx, _, err := DeployNameConflict(user, sim) + if err != nil { + t.Fatalf("DeployNameConflict() got err %v; want nil err", err) + } + sim.Commit() + + if _, err = bind.WaitDeployed(nil, sim, tx); err != nil { + t.Logf("Deployment tx: %+v", tx) + t.Errorf("bind.WaitDeployed(nil, %T, ) got err %v; want nil err", sim, err) + } + `, + }, + { + name: "RangeKeyword", + contract: ` + // SPDX-License-Identifier: GPL-3.0 + pragma solidity >=0.4.22 <0.9.0; + contract keywordcontract { + function functionWithKeywordParameter(range uint256) public pure {} + } + `, + bytecode: []string{"0x608060405234801561001057600080fd5b5060dc8061001f6000396000f3fe6080604052348015600f57600080fd5b506004361060285760003560e01c8063527a119f14602d575b600080fd5b60436004803603810190603f9190605b565b6045565b005b50565b6000813590506055816092565b92915050565b600060208284031215606e57606d608d565b5b6000607a848285016048565b91505092915050565b6000819050919050565b600080fd5b6099816083565b811460a357600080fd5b5056fea2646970667358221220d4f4525e2615516394055d369fb17df41c359e5e962734f27fd683ea81fd9db164736f6c63430008070033"}, + abi: []string{`[{"inputs":[{"internalType":"uint256","name":"range","type":"uint256"}],"name":"functionWithKeywordParameter","outputs":[],"stateMutability":"pure","type":"function"}]`}, + imports: ` + "math/big" + + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/eth/ethconfig" + `, + tester: ` + var ( + key, _ = crypto.GenerateKey() + user, _ = bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) + sim = backends.NewSimulatedBackend(core.GenesisAlloc{user.From: {Balance: big.NewInt(1000000000000000000)}}, ethconfig.Defaults.Miner.GasCeil) + ) + _, tx, _, err := DeployRangeKeyword(user, sim) + if err != nil { + t.Fatalf("error deploying contract: %v", err) + } + sim.Commit() + + if _, err = bind.WaitDeployed(nil, sim, tx); err != nil { + t.Errorf("error deploying the contract: %v", err) + } + `, + }, } // Tests that packages generated by the binder can be successfully compiled and diff --git a/accounts/abi/bind/template.go b/accounts/abi/bind/template.go index c9b001133dd51..855c8ead87ca4 100644 --- a/accounts/abi/bind/template.go +++ b/accounts/abi/bind/template.go @@ -110,6 +110,7 @@ var ( _ = common.Big1 _ = types.BloomLookup _ = event.NewSubscription + _ = abi.ConvertType ) {{$structs := .Structs}} @@ -268,11 +269,11 @@ var ( // bind{{.Type}} binds a generic wrapper to an already deployed contract. func bind{{.Type}}(address common.Address, caller bind.ContractCaller, transactor bind.ContractTransactor, filterer bind.ContractFilterer) (*bind.BoundContract, error) { - parsed, err := abi.JSON(strings.NewReader({{.Type}}ABI)) + parsed, err := {{.Type}}MetaData.GetAbi() if err != nil { return nil, err } - return bind.NewBoundContract(address, parsed, caller, transactor, filterer), nil + return bind.NewBoundContract(address, *parsed, caller, transactor, filterer), nil } // Call invokes the (constant) contract method with params as input values and diff --git a/accounts/abi/error.go b/accounts/abi/error.go index e564c10c2f840..f53c996def14d 100644 --- a/accounts/abi/error.go +++ b/accounts/abi/error.go @@ -30,11 +30,13 @@ type Error struct { Name string Inputs Arguments str string + // Sig contains the string signature according to the ABI spec. - // e.g. event foo(uint32 a, int b) = "foo(uint32,int256)" + // e.g. error foo(uint32 a, int b) = "foo(uint32,int256)" // Please note that "int" is substitute for its canonical representation "int256" Sig string - // ID returns the canonical representation of the event's signature used by the + + // ID returns the canonical representation of the error's signature used by the // abi definition to identify event names and types. ID common.Hash } diff --git a/accounts/abi/error_handling.go b/accounts/abi/error_handling.go index f0f71b6c91646..7add7072925e6 100644 --- a/accounts/abi/error_handling.go +++ b/accounts/abi/error_handling.go @@ -73,7 +73,6 @@ func typeCheck(t Type, value reflect.Value) error { } else { return nil } - } // typeErr returns a formatted type casting error. diff --git a/accounts/abi/event.go b/accounts/abi/event.go index b238a36d7ceab..f9457b86afebf 100644 --- a/accounts/abi/event.go +++ b/accounts/abi/event.go @@ -29,24 +29,27 @@ import ( // don't get the signature canonical representation as the first LOG topic. type Event struct { // Name is the event name used for internal representation. It's derived from - // the raw name and a suffix will be added in the case of a event overload. + // the raw name and a suffix will be added in the case of event overloading. // // e.g. // These are two events that have the same name: // * foo(int,int) // * foo(uint,uint) - // The event name of the first one wll be resolved as foo while the second one + // The event name of the first one will be resolved as foo while the second one // will be resolved as foo0. Name string + // RawName is the raw event name parsed from ABI. RawName string Anonymous bool Inputs Arguments str string + // Sig contains the string signature according to the ABI spec. // e.g. event foo(uint32 a, int b) = "foo(uint32,int256)" // Please note that "int" is substitute for its canonical representation "int256" Sig string + // ID returns the canonical representation of the event's signature used by the // abi definition to identify event names and types. ID common.Hash diff --git a/accounts/abi/event_test.go b/accounts/abi/event_test.go index 3332f8a07216c..8f73419496ba5 100644 --- a/accounts/abi/event_test.go +++ b/accounts/abi/event_test.go @@ -161,7 +161,6 @@ func TestEventMultiValueWithArrayUnpack(t *testing.T) { } func TestEventTupleUnpack(t *testing.T) { - type EventTransfer struct { Value *big.Int } diff --git a/accounts/abi/reflect.go b/accounts/abi/reflect.go index 35e5556d2c5a7..1f84b111a3db6 100644 --- a/accounts/abi/reflect.go +++ b/accounts/abi/reflect.go @@ -25,16 +25,19 @@ import ( ) // ConvertType converts an interface of a runtime type into a interface of the -// given type -// e.g. turn -// var fields []reflect.StructField -// fields = append(fields, reflect.StructField{ -// Name: "X", -// Type: reflect.TypeOf(new(big.Int)), -// Tag: reflect.StructTag("json:\"" + "x" + "\""), -// } -// into -// type TupleT struct { X *big.Int } +// given type, e.g. turn this code: +// +// var fields []reflect.StructField +// +// fields = append(fields, reflect.StructField{ +// Name: "X", +// Type: reflect.TypeOf(new(big.Int)), +// Tag: reflect.StructTag("json:\"" + "x" + "\""), +// } +// +// into: +// +// type TupleT struct { X *big.Int } func ConvertType(in interface{}, proto interface{}) interface{} { protoType := reflect.TypeOf(proto) if reflect.TypeOf(in).ConvertibleTo(protoType) { @@ -99,7 +102,7 @@ func mustArrayToByteSlice(value reflect.Value) reflect.Value { func set(dst, src reflect.Value) error { dstType, srcType := dst.Type(), src.Type() switch { - case dstType.Kind() == reflect.Interface && dst.Elem().IsValid(): + case dstType.Kind() == reflect.Interface && dst.Elem().IsValid() && (dst.Elem().Type().Kind() == reflect.Ptr || dst.Elem().CanSet()): return set(dst.Elem(), src) case dstType.Kind() == reflect.Ptr && dstType.Elem() != reflect.TypeOf(big.Int{}): return set(dst.Elem(), src) @@ -170,11 +173,13 @@ func setStruct(dst, src reflect.Value) error { } // mapArgNamesToStructFields maps a slice of argument names to struct fields. -// first round: for each Exportable field that contains a `abi:""` tag -// and this field name exists in the given argument name list, pair them together. -// second round: for each argument name that has not been already linked, -// find what variable is expected to be mapped into, if it exists and has not been -// used, pair them. +// +// first round: for each Exportable field that contains a `abi:""` tag and this field name +// exists in the given argument name list, pair them together. +// +// second round: for each argument name that has not been already linked, find what +// variable is expected to be mapped into, if it exists and has not been used, pair them. +// // Note this function assumes the given value is a struct value. func mapArgNamesToStructFields(argNames []string, value reflect.Value) (map[string]string, error) { typ := value.Type() @@ -220,7 +225,6 @@ func mapArgNamesToStructFields(argNames []string, value reflect.Value) (map[stri // second round ~~~ for _, argName := range argNames { - structFieldName := ToCamelCase(argName) if structFieldName == "" { diff --git a/accounts/abi/reflect_test.go b/accounts/abi/reflect_test.go index cf13a79da84e6..76ef1ad2aa39b 100644 --- a/accounts/abi/reflect_test.go +++ b/accounts/abi/reflect_test.go @@ -32,7 +32,7 @@ type reflectTest struct { var reflectTests = []reflectTest{ { - name: "OneToOneCorrespondance", + name: "OneToOneCorrespondence", args: []string{"fieldA"}, struc: struct { FieldA int `abi:"fieldA"` diff --git a/accounts/abi/selector_parser.go b/accounts/abi/selector_parser.go index 88114e288eb39..d5472e374f5dd 100644 --- a/accounts/abi/selector_parser.go +++ b/accounts/abi/selector_parser.go @@ -166,7 +166,7 @@ func ParseSelector(unescapedSelector string) (SelectorMarshaling, error) { return SelectorMarshaling{}, fmt.Errorf("failed to parse selector '%s': unexpected string '%s'", unescapedSelector, rest) } - // Reassemble the fake ABI and constuct the JSON + // Reassemble the fake ABI and construct the JSON fakeArgs, err := assembleArgs(args) if err != nil { return SelectorMarshaling{}, fmt.Errorf("failed to parse selector: %v", err) diff --git a/accounts/abi/type.go b/accounts/abi/type.go index fd75f586a99db..008b665b1aeeb 100644 --- a/accounts/abi/type.go +++ b/accounts/abi/type.go @@ -163,22 +163,26 @@ func NewType(t string, internalType string, components []ArgumentMarshaling) (ty elems []*Type names []string expression string // canonical parameter expression + used = make(map[string]bool) ) expression += "(" - overloadedNames := make(map[string]string) for idx, c := range components { cType, err := NewType(c.Type, c.InternalType, c.Components) if err != nil { return Type{}, err } - fieldName, err := overloadedArgName(c.Name, overloadedNames) + name := ToCamelCase(c.Name) + if name == "" { + return Type{}, errors.New("abi: purely anonymous or underscored field is not supported") + } + fieldName := ResolveNameConflict(name, func(s string) bool { return used[s] }) if err != nil { return Type{}, err } + used[fieldName] = true if !isValidFieldName(fieldName) { return Type{}, fmt.Errorf("field %d has invalid name", idx) } - overloadedNames[fieldName] = fieldName fields = append(fields, reflect.StructField{ Name: fieldName, // reflect.StructOf will panic for any exported field. Type: cType.GetType(), @@ -255,20 +259,6 @@ func (t Type) GetType() reflect.Type { } } -func overloadedArgName(rawName string, names map[string]string) (string, error) { - fieldName := ToCamelCase(rawName) - if fieldName == "" { - return "", errors.New("abi: purely anonymous or underscored field is not supported") - } - // Handle overloaded fieldNames - _, ok := names[fieldName] - for idx := 0; ok; idx++ { - fieldName = fmt.Sprintf("%s%d", ToCamelCase(rawName), idx) - _, ok = names[fieldName] - } - return fieldName, nil -} - // String implements Stringer. func (t Type) String() (out string) { return t.stringKind diff --git a/accounts/abi/unpack.go b/accounts/abi/unpack.go index 43cd6c64575c7..b6ca0a0384806 100644 --- a/accounts/abi/unpack.go +++ b/accounts/abi/unpack.go @@ -115,7 +115,6 @@ func ReadFixedBytes(t Type, word []byte) (interface{}, error) { reflect.Copy(array, reflect.ValueOf(word[0:t.Size])) return array.Interface(), nil - } // forEachUnpack iteratively unpack elements. @@ -124,7 +123,7 @@ func forEachUnpack(t Type, output []byte, start, size int) (interface{}, error) return nil, fmt.Errorf("cannot marshal input to array, size is negative (%d)", size) } if start+32*size > len(output) { - return nil, fmt.Errorf("abi: cannot marshal in to go array: offset %d would go over slice boundary (len=%d)", len(output), start+32*size) + return nil, fmt.Errorf("abi: cannot marshal into go array: offset %d would go over slice boundary (len=%d)", len(output), start+32*size) } // this value will become our slice or our array, depending on the type @@ -163,6 +162,9 @@ func forTupleUnpack(t Type, output []byte) (interface{}, error) { virtualArgs := 0 for index, elem := range t.TupleElems { marshalledValue, err := toGoType((index+virtualArgs)*32, *elem, output) + if err != nil { + return nil, err + } if elem.T == ArrayTy && !isDynamicType(*elem) { // If we have a static array, like [3]uint256, these are coded as // just like uint256,uint256,uint256. @@ -180,9 +182,6 @@ func forTupleUnpack(t Type, output []byte) (interface{}, error) { // coded as just like uint256,bool,uint256 virtualArgs += getTypeSize(*elem)/32 - 1 } - if err != nil { - return nil, err - } retval.Field(index).Set(reflect.ValueOf(marshalledValue)) } return retval.Interface(), nil @@ -255,7 +254,7 @@ func toGoType(index int, t Type, output []byte) (interface{}, error) { // lengthPrefixPointsTo interprets a 32 byte slice as an offset and then determines which indices to look to decode the type. func lengthPrefixPointsTo(index int, output []byte) (start int, length int, err error) { - bigOffsetEnd := big.NewInt(0).SetBytes(output[index : index+32]) + bigOffsetEnd := new(big.Int).SetBytes(output[index : index+32]) bigOffsetEnd.Add(bigOffsetEnd, common.Big32) outputLength := big.NewInt(int64(len(output))) @@ -268,11 +267,9 @@ func lengthPrefixPointsTo(index int, output []byte) (start int, length int, err } offsetEnd := int(bigOffsetEnd.Uint64()) - lengthBig := big.NewInt(0).SetBytes(output[offsetEnd-32 : offsetEnd]) + lengthBig := new(big.Int).SetBytes(output[offsetEnd-32 : offsetEnd]) - totalSize := big.NewInt(0) - totalSize.Add(totalSize, bigOffsetEnd) - totalSize.Add(totalSize, lengthBig) + totalSize := new(big.Int).Add(bigOffsetEnd, lengthBig) if totalSize.BitLen() > 63 { return 0, 0, fmt.Errorf("abi: length larger than int64: %v", totalSize) } @@ -287,7 +284,7 @@ func lengthPrefixPointsTo(index int, output []byte) (start int, length int, err // tuplePointsTo resolves the location reference for dynamic tuple. func tuplePointsTo(index int, output []byte) (start int, err error) { - offset := big.NewInt(0).SetBytes(output[index : index+32]) + offset := new(big.Int).SetBytes(output[index : index+32]) outputLen := big.NewInt(int64(len(output))) if offset.Cmp(outputLen) > 0 { diff --git a/accounts/abi/unpack_test.go b/accounts/abi/unpack_test.go index bf40c301b5f7c..363e0cd5943ec 100644 --- a/accounts/abi/unpack_test.go +++ b/accounts/abi/unpack_test.go @@ -352,6 +352,11 @@ func TestMethodMultiReturn(t *testing.T) { &[]interface{}{&expected.Int, &expected.String}, "", "Can unpack into a slice", + }, { + &[]interface{}{&bigint, ""}, + &[]interface{}{&expected.Int, expected.String}, + "", + "Can unpack into a slice without indirection", }, { &[2]interface{}{&bigint, new(string)}, &[2]interface{}{&expected.Int, &expected.String}, @@ -424,7 +429,7 @@ func TestMultiReturnWithStringArray(t *testing.T) { } buff := new(bytes.Buffer) buff.Write(common.Hex2Bytes("000000000000000000000000000000000000000000000000000000005c1b78ea0000000000000000000000000000000000000000000000000000000000000006000000000000000000000000000000000000000000000001a055690d9db80000000000000000000000000000ab1257528b3782fb40d7ed5f72e624b744dffb2f00000000000000000000000000000000000000000000000000000000000000c00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000008457468657265756d000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001048656c6c6f2c20457468657265756d2100000000000000000000000000000000")) - temp, _ := big.NewInt(0).SetString("30000000000000000000", 10) + temp, _ := new(big.Int).SetString("30000000000000000000", 10) ret1, ret1Exp := new([3]*big.Int), [3]*big.Int{big.NewInt(1545304298), big.NewInt(6), temp} ret2, ret2Exp := new(common.Address), common.HexToAddress("ab1257528b3782fb40d7ed5f72e624b744dffb2f") ret3, ret3Exp := new([2]string), [2]string{"Ethereum", "Hello, Ethereum!"} diff --git a/accounts/abi/utils.go b/accounts/abi/utils.go new file mode 100644 index 0000000000000..b1537ca58dd32 --- /dev/null +++ b/accounts/abi/utils.go @@ -0,0 +1,40 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package abi + +import "fmt" + +// ResolveNameConflict returns the next available name for a given thing. +// This helper can be used for lots of purposes: +// +// - In solidity function overloading is supported, this function can fix +// the name conflicts of overloaded functions. +// - In golang binding generation, the parameter(in function, event, error, +// and struct definition) name will be converted to camelcase style which +// may eventually lead to name conflicts. +// +// Name conflicts are mostly resolved by adding number suffix. e.g. if the abi contains +// Methods "send" and "send1", ResolveNameConflict would return "send2" for input "send". +func ResolveNameConflict(rawName string, used func(string) bool) string { + name := rawName + ok := used(name) + for idx := 0; ok; idx++ { + name = fmt.Sprintf("%s%d", rawName, idx) + ok = used(name) + } + return name +} diff --git a/accounts/accounts.go b/accounts/accounts.go index 179a33c59fd32..6c351a9649ea0 100644 --- a/accounts/accounts.go +++ b/accounts/accounts.go @@ -177,7 +177,8 @@ type Backend interface { // safely used to calculate a signature from. // // The hash is calculated as -// keccak256("\x19Ethereum Signed Message:\n"${message length}${message}). +// +// keccak256("\x19Ethereum Signed Message:\n"${message length}${message}). // // This gives context to the signed message and prevents signing of transactions. func TextHash(data []byte) []byte { @@ -189,7 +190,8 @@ func TextHash(data []byte) []byte { // safely used to calculate a signature from. // // The hash is calculated as -// keccak256("\x19Ethereum Signed Message:\n"${message length}${message}). +// +// keccak256("\x19Ethereum Signed Message:\n"${message length}${message}). // // This gives context to the signed message and prevents signing of transactions. func TextAndHash(data []byte) ([]byte, string) { diff --git a/accounts/external/backend.go b/accounts/external/backend.go index e3f754eafcc42..d403b7e562d41 100644 --- a/accounts/external/backend.go +++ b/accounts/external/backend.go @@ -152,10 +152,6 @@ func (api *ExternalSigner) SelfDerive(bases []accounts.DerivationPath, chain eth log.Error("operation SelfDerive not supported on external signers") } -func (api *ExternalSigner) signHash(account accounts.Account, hash []byte) ([]byte, error) { - return []byte{}, fmt.Errorf("operation not supported on external signers") -} - // SignData signs keccak256(data). The mimetype parameter describes the type of data being signed func (api *ExternalSigner) SignData(account accounts.Account, mimeType string, data []byte) ([]byte, error) { var res hexutil.Bytes diff --git a/accounts/hd.go b/accounts/hd.go index 3009f19b65778..daca75ebbcb71 100644 --- a/accounts/hd.go +++ b/accounts/hd.go @@ -46,7 +46,7 @@ var LegacyLedgerBaseDerivationPath = DerivationPath{0x80000000 + 44, 0x80000000 // The BIP-32 spec https://github.com/bitcoin/bips/blob/master/bip-0032.mediawiki // defines derivation paths to be of the form: // -// m / purpose' / coin_type' / account' / change / address_index +// m / purpose' / coin_type' / account' / change / address_index // // The BIP-44 spec https://github.com/bitcoin/bips/blob/master/bip-0044.mediawiki // defines that the `purpose` be 44' (or 0x8000002C) for crypto currencies, and diff --git a/accounts/keystore/account_cache_test.go b/accounts/keystore/account_cache_test.go index fda0e5667c2aa..daea497d1ae76 100644 --- a/accounts/keystore/account_cache_test.go +++ b/accounts/keystore/account_cache_test.go @@ -318,7 +318,7 @@ func waitForAccounts(wantAccounts []accounts.Account, ks *KeyStore) error { func TestUpdatedKeyfileContents(t *testing.T) { t.Parallel() - // Create a temporary kesytore to test with + // Create a temporary keystore to test with rand.Seed(time.Now().UnixNano()) dir := filepath.Join(os.TempDir(), fmt.Sprintf("eth-keystore-updatedkeyfilecontents-test-%d-%d", os.Getpid(), rand.Int())) ks := NewKeyStore(dir, LightScryptN, LightScryptP) @@ -383,7 +383,7 @@ func TestUpdatedKeyfileContents(t *testing.T) { time.Sleep(1000 * time.Millisecond) // Now replace file contents with crap - if err := os.WriteFile(file, []byte("foo"), 0644); err != nil { + if err := os.WriteFile(file, []byte("foo"), 0600); err != nil { t.Fatal(err) return } diff --git a/accounts/keystore/file_cache.go b/accounts/keystore/file_cache.go index b3ecf8946b53b..79f9a29637431 100644 --- a/accounts/keystore/file_cache.go +++ b/accounts/keystore/file_cache.go @@ -39,7 +39,7 @@ type fileCache struct { func (fc *fileCache) scan(keyDir string) (mapset.Set, mapset.Set, mapset.Set, error) { t0 := time.Now() - // List all the failes from the keystore folder + // List all the files from the keystore folder files, err := os.ReadDir(keyDir) if err != nil { return nil, nil, nil, err @@ -61,7 +61,7 @@ func (fc *fileCache) scan(keyDir string) (mapset.Set, mapset.Set, mapset.Set, er log.Trace("Ignoring file on account scan", "path", path) continue } - // Gather the set of all and fresly modified files + // Gather the set of all and freshly modified files all.Add(path) info, err := fi.Info() diff --git a/accounts/keystore/keystore_test.go b/accounts/keystore/keystore_test.go index 80c4d643e8e18..4cdf0b1ed6ce8 100644 --- a/accounts/keystore/keystore_test.go +++ b/accounts/keystore/keystore_test.go @@ -214,7 +214,7 @@ func TestSignRace(t *testing.T) { // Tests that the wallet notifier loop starts and stops correctly based on the // addition and removal of wallet event subscriptions. func TestWalletNotifierLifecycle(t *testing.T) { - // Create a temporary kesytore to test with + // Create a temporary keystore to test with _, ks := tmpKeyStore(t, false) // Ensure that the notification updater is not running yet @@ -377,7 +377,6 @@ func TestImportExport(t *testing.T) { if _, err = ks2.Import(json, "new", "new"); err == nil { t.Errorf("importing a key twice succeeded") } - } // TestImportRace tests the keystore on races. @@ -402,7 +401,6 @@ func TestImportRace(t *testing.T) { if _, err := ks2.Import(json, "new", "new"); err != nil { atomic.AddUint32(&atom, 1) } - }() } wg.Wait() diff --git a/accounts/keystore/passphrase.go b/accounts/keystore/passphrase.go index 22772e93102f4..1701fbf53634e 100644 --- a/accounts/keystore/passphrase.go +++ b/accounts/keystore/passphrase.go @@ -138,7 +138,6 @@ func (ks keyStorePassphrase) JoinPath(filename string) string { // Encryptdata encrypts the data given as 'data' with the password 'auth'. func EncryptDataV3(data, auth []byte, scryptN, scryptP int) (CryptoJSON, error) { - salt := make([]byte, 32) if _, err := io.ReadFull(rand.Reader, salt); err != nil { panic("reading from crypto/rand failed: " + err.Error()) @@ -341,7 +340,6 @@ func getKDFKey(cryptoJSON CryptoJSON, auth string) ([]byte, error) { r := ensureInt(cryptoJSON.KDFParams["r"]) p := ensureInt(cryptoJSON.KDFParams["p"]) return scrypt.Key(authArray, salt, n, r, p, dkLen) - } else if cryptoJSON.KDF == "pbkdf2" { c := ensureInt(cryptoJSON.KDFParams["c"]) prf := cryptoJSON.KDFParams["prf"].(string) diff --git a/accounts/keystore/passphrase_test.go b/accounts/keystore/passphrase_test.go index b94fce8edcae7..1356b317806dc 100644 --- a/accounts/keystore/passphrase_test.go +++ b/accounts/keystore/passphrase_test.go @@ -52,7 +52,7 @@ func TestKeyEncryptDecrypt(t *testing.T) { t.Errorf("test %d: key address mismatch: have %x, want %x", i, key.Address, address) } // Recrypt with a new password and start over - password += "new data appended" + password += "new data appended" // nolint: gosec if keyjson, err = EncryptKey(key, password, veryLightScryptN, veryLightScryptP); err != nil { t.Errorf("test %d: failed to recrypt key %v", i, err) } diff --git a/accounts/scwallet/securechannel.go b/accounts/scwallet/securechannel.go index 10887a8b43d0f..b1b533eb72438 100644 --- a/accounts/scwallet/securechannel.go +++ b/accounts/scwallet/securechannel.go @@ -178,7 +178,7 @@ func (s *SecureChannelSession) mutuallyAuthenticate() error { return err } if response.Sw1 != 0x90 || response.Sw2 != 0x00 { - return fmt.Errorf("got unexpected response from MUTUALLY_AUTHENTICATE: 0x%x%x", response.Sw1, response.Sw2) + return fmt.Errorf("got unexpected response from MUTUALLY_AUTHENTICATE: %#x%x", response.Sw1, response.Sw2) } if len(response.Data) != scSecretLength { @@ -261,7 +261,7 @@ func (s *SecureChannelSession) transmitEncrypted(cla, ins, p1, p2 byte, data []b rapdu.deserialize(plainData) if rapdu.Sw1 != sw1Ok { - return nil, fmt.Errorf("unexpected response status Cla=0x%x, Ins=0x%x, Sw=0x%x%x", cla, ins, rapdu.Sw1, rapdu.Sw2) + return nil, fmt.Errorf("unexpected response status Cla=%#x, Ins=%#x, Sw=%#x%x", cla, ins, rapdu.Sw1, rapdu.Sw2) } return rapdu, nil diff --git a/accounts/scwallet/wallet.go b/accounts/scwallet/wallet.go index 2a2b83bd1b156..e66717c3b1ade 100644 --- a/accounts/scwallet/wallet.go +++ b/accounts/scwallet/wallet.go @@ -167,7 +167,7 @@ func transmit(card *pcsc.Card, command *commandAPDU) (*responseAPDU, error) { } if response.Sw1 != sw1Ok { - return nil, fmt.Errorf("unexpected insecure response status Cla=0x%x, Ins=0x%x, Sw=0x%x%x", command.Cla, command.Ins, response.Sw1, response.Sw2) + return nil, fmt.Errorf("unexpected insecure response status Cla=%#x, Ins=%#x, Sw=%#x%x", command.Cla, command.Ins, response.Sw1, response.Sw2) } return response, nil @@ -879,6 +879,7 @@ func (s *Session) walletStatus() (*walletStatus, error) { } // derivationPath fetches the wallet's current derivation path from the card. +// //lint:ignore U1000 needs to be added to the console interface func (s *Session) derivationPath() (accounts.DerivationPath, error) { response, err := s.Channel.transmitEncrypted(claSCWallet, insStatus, statusP1Path, 0, nil) @@ -994,6 +995,7 @@ func (s *Session) derive(path accounts.DerivationPath) (accounts.Account, error) } // keyExport contains information on an exported keypair. +// //lint:ignore U1000 needs to be added to the console interface type keyExport struct { PublicKey []byte `asn1:"tag:0"` @@ -1001,6 +1003,7 @@ type keyExport struct { } // publicKey returns the public key for the current derivation path. +// //lint:ignore U1000 needs to be added to the console interface func (s *Session) publicKey() ([]byte, error) { response, err := s.Channel.transmitEncrypted(claSCWallet, insExportKey, exportP1Any, exportP2Pubkey, nil) diff --git a/accounts/url.go b/accounts/url.go index 12a84414a057b..39b00e5b4498f 100644 --- a/accounts/url.go +++ b/accounts/url.go @@ -92,10 +92,9 @@ func (u *URL) UnmarshalJSON(input []byte) error { // Cmp compares x and y and returns: // -// -1 if x < y -// 0 if x == y -// +1 if x > y -// +// -1 if x < y +// 0 if x == y +// +1 if x > y func (u URL) Cmp(url URL) int { if u.Scheme == url.Scheme { return strings.Compare(u.Path, url.Path) diff --git a/accounts/url_test.go b/accounts/url_test.go index bd6f35fa2a0ea..239aa06d227ba 100644 --- a/accounts/url_test.go +++ b/accounts/url_test.go @@ -32,9 +32,10 @@ func TestURLParsing(t *testing.T) { t.Errorf("expected: %v, got: %v", "ethereum.org", url.Path) } - _, err = parseURL("ethereum.org") - if err == nil { - t.Error("expected err, got: nil") + for _, u := range []string{"ethereum.org", ""} { + if _, err = parseURL(u); err == nil { + t.Errorf("input %v, expected err, got: nil", u) + } } } diff --git a/accounts/usbwallet/ledger.go b/accounts/usbwallet/ledger.go index 3de3b4091cfce..cda94280fdd20 100644 --- a/accounts/usbwallet/ledger.go +++ b/accounts/usbwallet/ledger.go @@ -195,18 +195,18 @@ func (w *ledgerDriver) SignTypedMessage(path accounts.DerivationPath, domainHash // // The version retrieval protocol is defined as follows: // -// CLA | INS | P1 | P2 | Lc | Le -// ----+-----+----+----+----+--- -// E0 | 06 | 00 | 00 | 00 | 04 +// CLA | INS | P1 | P2 | Lc | Le +// ----+-----+----+----+----+--- +// E0 | 06 | 00 | 00 | 00 | 04 // // With no input data, and the output data being: // -// Description | Length -// ---------------------------------------------------+-------- -// Flags 01: arbitrary data signature enabled by user | 1 byte -// Application major version | 1 byte -// Application minor version | 1 byte -// Application patch version | 1 byte +// Description | Length +// ---------------------------------------------------+-------- +// Flags 01: arbitrary data signature enabled by user | 1 byte +// Application major version | 1 byte +// Application minor version | 1 byte +// Application patch version | 1 byte func (w *ledgerDriver) ledgerVersion() ([3]byte, error) { // Send the request and wait for the response reply, err := w.ledgerExchange(ledgerOpGetConfiguration, 0, 0, nil) @@ -227,32 +227,32 @@ func (w *ledgerDriver) ledgerVersion() ([3]byte, error) { // // The address derivation protocol is defined as follows: // -// CLA | INS | P1 | P2 | Lc | Le -// ----+-----+----+----+-----+--- -// E0 | 02 | 00 return address -// 01 display address and confirm before returning -// | 00: do not return the chain code -// | 01: return the chain code -// | var | 00 +// CLA | INS | P1 | P2 | Lc | Le +// ----+-----+----+----+-----+--- +// E0 | 02 | 00 return address +// 01 display address and confirm before returning +// | 00: do not return the chain code +// | 01: return the chain code +// | var | 00 // // Where the input data is: // -// Description | Length -// -------------------------------------------------+-------- -// Number of BIP 32 derivations to perform (max 10) | 1 byte -// First derivation index (big endian) | 4 bytes -// ... | 4 bytes -// Last derivation index (big endian) | 4 bytes +// Description | Length +// -------------------------------------------------+-------- +// Number of BIP 32 derivations to perform (max 10) | 1 byte +// First derivation index (big endian) | 4 bytes +// ... | 4 bytes +// Last derivation index (big endian) | 4 bytes // // And the output data is: // -// Description | Length -// ------------------------+------------------- -// Public Key length | 1 byte -// Uncompressed Public Key | arbitrary -// Ethereum address length | 1 byte -// Ethereum address | 40 bytes hex ascii -// Chain code if requested | 32 bytes +// Description | Length +// ------------------------+------------------- +// Public Key length | 1 byte +// Uncompressed Public Key | arbitrary +// Ethereum address length | 1 byte +// Ethereum address | 40 bytes hex ascii +// Chain code if requested | 32 bytes func (w *ledgerDriver) ledgerDerive(derivationPath []uint32) (common.Address, error) { // Flatten the derivation path into the Ledger request path := make([]byte, 1+4*len(derivationPath)) @@ -290,35 +290,35 @@ func (w *ledgerDriver) ledgerDerive(derivationPath []uint32) (common.Address, er // // The transaction signing protocol is defined as follows: // -// CLA | INS | P1 | P2 | Lc | Le -// ----+-----+----+----+-----+--- -// E0 | 04 | 00: first transaction data block -// 80: subsequent transaction data block -// | 00 | variable | variable +// CLA | INS | P1 | P2 | Lc | Le +// ----+-----+----+----+-----+--- +// E0 | 04 | 00: first transaction data block +// 80: subsequent transaction data block +// | 00 | variable | variable // // Where the input for the first transaction block (first 255 bytes) is: // -// Description | Length -// -------------------------------------------------+---------- -// Number of BIP 32 derivations to perform (max 10) | 1 byte -// First derivation index (big endian) | 4 bytes -// ... | 4 bytes -// Last derivation index (big endian) | 4 bytes -// RLP transaction chunk | arbitrary +// Description | Length +// -------------------------------------------------+---------- +// Number of BIP 32 derivations to perform (max 10) | 1 byte +// First derivation index (big endian) | 4 bytes +// ... | 4 bytes +// Last derivation index (big endian) | 4 bytes +// RLP transaction chunk | arbitrary // // And the input for subsequent transaction blocks (first 255 bytes) are: // -// Description | Length -// ----------------------+---------- -// RLP transaction chunk | arbitrary +// Description | Length +// ----------------------+---------- +// RLP transaction chunk | arbitrary // // And the output data is: // -// Description | Length -// ------------+--------- -// signature V | 1 byte -// signature R | 32 bytes -// signature S | 32 bytes +// Description | Length +// ------------+--------- +// signature V | 1 byte +// signature R | 32 bytes +// signature S | 32 bytes func (w *ledgerDriver) ledgerSign(derivationPath []uint32, tx *types.Transaction, chainID *big.Int) (common.Address, *types.Transaction, error) { // Flatten the derivation path into the Ledger request path := make([]byte, 1+4*len(derivationPath)) @@ -392,30 +392,28 @@ func (w *ledgerDriver) ledgerSign(derivationPath []uint32, tx *types.Transaction // // The signing protocol is defined as follows: // -// CLA | INS | P1 | P2 | Lc | Le -// ----+-----+----+-----------------------------+-----+--- -// E0 | 0C | 00 | implementation version : 00 | variable | variable +// CLA | INS | P1 | P2 | Lc | Le +// ----+-----+----+-----------------------------+-----+--- +// E0 | 0C | 00 | implementation version : 00 | variable | variable // // Where the input is: // -// Description | Length -// -------------------------------------------------+---------- -// Number of BIP 32 derivations to perform (max 10) | 1 byte -// First derivation index (big endian) | 4 bytes -// ... | 4 bytes -// Last derivation index (big endian) | 4 bytes -// domain hash | 32 bytes -// message hash | 32 bytes -// -// +// Description | Length +// -------------------------------------------------+---------- +// Number of BIP 32 derivations to perform (max 10) | 1 byte +// First derivation index (big endian) | 4 bytes +// ... | 4 bytes +// Last derivation index (big endian) | 4 bytes +// domain hash | 32 bytes +// message hash | 32 bytes // // And the output data is: // -// Description | Length -// ------------+--------- -// signature V | 1 byte -// signature R | 32 bytes -// signature S | 32 bytes +// Description | Length +// ------------+--------- +// signature V | 1 byte +// signature R | 32 bytes +// signature S | 32 bytes func (w *ledgerDriver) ledgerSignTypedMessage(derivationPath []uint32, domainHash []byte, messageHash []byte) ([]byte, error) { // Flatten the derivation path into the Ledger request path := make([]byte, 1+4*len(derivationPath)) @@ -454,12 +452,12 @@ func (w *ledgerDriver) ledgerSignTypedMessage(derivationPath []uint32, domainHas // // The common transport header is defined as follows: // -// Description | Length -// --------------------------------------+---------- -// Communication channel ID (big endian) | 2 bytes -// Command tag | 1 byte -// Packet sequence index (big endian) | 2 bytes -// Payload | arbitrary +// Description | Length +// --------------------------------------+---------- +// Communication channel ID (big endian) | 2 bytes +// Command tag | 1 byte +// Packet sequence index (big endian) | 2 bytes +// Payload | arbitrary // // The Communication channel ID allows commands multiplexing over the same // physical link. It is not used for the time being, and should be set to 0101 @@ -473,15 +471,15 @@ func (w *ledgerDriver) ledgerSignTypedMessage(derivationPath []uint32, domainHas // // APDU Command payloads are encoded as follows: // -// Description | Length -// ----------------------------------- -// APDU length (big endian) | 2 bytes -// APDU CLA | 1 byte -// APDU INS | 1 byte -// APDU P1 | 1 byte -// APDU P2 | 1 byte -// APDU length | 1 byte -// Optional APDU data | arbitrary +// Description | Length +// ----------------------------------- +// APDU length (big endian) | 2 bytes +// APDU CLA | 1 byte +// APDU INS | 1 byte +// APDU P1 | 1 byte +// APDU P2 | 1 byte +// APDU length | 1 byte +// Optional APDU data | arbitrary func (w *ledgerDriver) ledgerExchange(opcode ledgerOpcode, p1 ledgerParam1, p2 ledgerParam2, data []byte) ([]byte, error) { // Construct the message payload, possibly split into multiple chunks apdu := make([]byte, 2, 7+len(data)) diff --git a/accounts/usbwallet/trezor.go b/accounts/usbwallet/trezor.go index c2182b88d03be..9644dc4e02c9c 100644 --- a/accounts/usbwallet/trezor.go +++ b/accounts/usbwallet/trezor.go @@ -84,15 +84,15 @@ func (w *trezorDriver) Status() (string, error) { // Open implements usbwallet.driver, attempting to initialize the connection to // the Trezor hardware wallet. Initializing the Trezor is a two or three phase operation: -// * The first phase is to initialize the connection and read the wallet's -// features. This phase is invoked if the provided passphrase is empty. The -// device will display the pinpad as a result and will return an appropriate -// error to notify the user that a second open phase is needed. -// * The second phase is to unlock access to the Trezor, which is done by the -// user actually providing a passphrase mapping a keyboard keypad to the pin -// number of the user (shuffled according to the pinpad displayed). -// * If needed the device will ask for passphrase which will require calling -// open again with the actual passphrase (3rd phase) +// - The first phase is to initialize the connection and read the wallet's +// features. This phase is invoked if the provided passphrase is empty. The +// device will display the pinpad as a result and will return an appropriate +// error to notify the user that a second open phase is needed. +// - The second phase is to unlock access to the Trezor, which is done by the +// user actually providing a passphrase mapping a keyboard keypad to the pin +// number of the user (shuffled according to the pinpad displayed). +// - If needed the device will ask for passphrase which will require calling +// open again with the actual passphrase (3rd phase) func (w *trezorDriver) Open(device io.ReadWriter, passphrase string) error { w.device, w.failure = device, nil @@ -196,10 +196,10 @@ func (w *trezorDriver) trezorDerive(derivationPath []uint32) (common.Address, er if _, err := w.trezorExchange(&trezor.EthereumGetAddress{AddressN: derivationPath}, address); err != nil { return common.Address{}, err } - if addr := address.GetAddressBin(); len(addr) > 0 { // Older firmwares use binary fomats + if addr := address.GetAddressBin(); len(addr) > 0 { // Older firmwares use binary formats return common.BytesToAddress(addr), nil } - if addr := address.GetAddressHex(); len(addr) > 0 { // Newer firmwares use hexadecimal fomats + if addr := address.GetAddressHex(); len(addr) > 0 { // Newer firmwares use hexadecimal formats return common.HexToAddress(addr), nil } return common.Address{}, errors.New("missing derived address") diff --git a/accounts/usbwallet/trezor/messages-common.pb.go b/accounts/usbwallet/trezor/messages-common.pb.go index 304bec0e360a1..b396c6d8b5546 100644 --- a/accounts/usbwallet/trezor/messages-common.pb.go +++ b/accounts/usbwallet/trezor/messages-common.pb.go @@ -94,7 +94,7 @@ func (Failure_FailureType) EnumDescriptor() ([]byte, []int) { return fileDescriptor_aaf30d059fdbc38d, []int{1, 0} } -//* +// * // Type of button request type ButtonRequest_ButtonRequestType int32 @@ -175,7 +175,7 @@ func (ButtonRequest_ButtonRequestType) EnumDescriptor() ([]byte, []int) { return fileDescriptor_aaf30d059fdbc38d, []int{2, 0} } -//* +// * // Type of PIN request type PinMatrixRequest_PinMatrixRequestType int32 @@ -220,7 +220,7 @@ func (PinMatrixRequest_PinMatrixRequestType) EnumDescriptor() ([]byte, []int) { return fileDescriptor_aaf30d059fdbc38d, []int{4, 0} } -//* +// * // Response: Success of the previous request // @end type Success struct { @@ -262,7 +262,7 @@ func (m *Success) GetMessage() string { return "" } -//* +// * // Response: Failure of the previous request // @end type Failure struct { @@ -312,7 +312,7 @@ func (m *Failure) GetMessage() string { return "" } -//* +// * // Response: Device is waiting for HW button press. // @auxstart // @next ButtonAck @@ -363,7 +363,7 @@ func (m *ButtonRequest) GetData() string { return "" } -//* +// * // Request: Computer agrees to wait for HW button press // @auxend type ButtonAck struct { @@ -397,7 +397,7 @@ func (m *ButtonAck) XXX_DiscardUnknown() { var xxx_messageInfo_ButtonAck proto.InternalMessageInfo -//* +// * // Response: Device is asking computer to show PIN matrix and awaits PIN encoded using this matrix scheme // @auxstart // @next PinMatrixAck @@ -440,7 +440,7 @@ func (m *PinMatrixRequest) GetType() PinMatrixRequest_PinMatrixRequestType { return PinMatrixRequest_PinMatrixRequestType_Current } -//* +// * // Request: Computer responds with encoded PIN // @auxend type PinMatrixAck struct { @@ -482,7 +482,7 @@ func (m *PinMatrixAck) GetPin() string { return "" } -//* +// * // Response: Device awaits encryption passphrase // @auxstart // @next PassphraseAck @@ -525,7 +525,7 @@ func (m *PassphraseRequest) GetOnDevice() bool { return false } -//* +// * // Request: Send passphrase back // @next PassphraseStateRequest type PassphraseAck struct { @@ -575,7 +575,7 @@ func (m *PassphraseAck) GetState() []byte { return nil } -//* +// * // Response: Device awaits passphrase state // @next PassphraseStateAck type PassphraseStateRequest struct { @@ -617,7 +617,7 @@ func (m *PassphraseStateRequest) GetState() []byte { return nil } -//* +// * // Request: Send passphrase state back // @auxend type PassphraseStateAck struct { @@ -651,7 +651,7 @@ func (m *PassphraseStateAck) XXX_DiscardUnknown() { var xxx_messageInfo_PassphraseStateAck proto.InternalMessageInfo -//* +// * // Structure representing BIP32 (hierarchical deterministic) node // Used for imports of private key into the device and exporting public key out of device // @embed diff --git a/accounts/usbwallet/trezor/messages-ethereum.pb.go b/accounts/usbwallet/trezor/messages-ethereum.pb.go index 5d664f5ba447a..230a48279d48d 100644 --- a/accounts/usbwallet/trezor/messages-ethereum.pb.go +++ b/accounts/usbwallet/trezor/messages-ethereum.pb.go @@ -21,7 +21,7 @@ var _ = math.Inf // proto package needs to be updated. const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package -//* +// * // Request: Ask device for public key corresponding to address_n path // @start // @next EthereumPublicKey @@ -73,7 +73,7 @@ func (m *EthereumGetPublicKey) GetShowDisplay() bool { return false } -//* +// * // Response: Contains public key derived from device private seed // @end type EthereumPublicKey struct { @@ -123,7 +123,7 @@ func (m *EthereumPublicKey) GetXpub() string { return "" } -//* +// * // Request: Ask device for Ethereum address corresponding to address_n path // @start // @next EthereumAddress @@ -175,7 +175,7 @@ func (m *EthereumGetAddress) GetShowDisplay() bool { return false } -//* +// * // Response: Contains an Ethereum address derived from device private seed // @end type EthereumAddress struct { @@ -225,7 +225,7 @@ func (m *EthereumAddress) GetAddressHex() string { return "" } -//* +// * // Request: Ask device to sign transaction // All fields are optional from the protocol's point of view. Each field defaults to value `0` if missing. // Note: the first at most 1024 bytes of data MUST be transmitted as part of this message. @@ -351,7 +351,7 @@ func (m *EthereumSignTx) GetTxType() uint32 { return 0 } -//* +// * // Response: Device asks for more data from transaction payload, or returns the signature. // If data_length is set, device awaits that many more bytes of payload. // Otherwise, the signature_* fields contain the computed transaction signature. All three fields will be present. @@ -420,7 +420,7 @@ func (m *EthereumTxRequest) GetSignatureS() []byte { return nil } -//* +// * // Request: Transaction payload data. // @next EthereumTxRequest type EthereumTxAck struct { @@ -462,7 +462,7 @@ func (m *EthereumTxAck) GetDataChunk() []byte { return nil } -//* +// * // Request: Ask device to sign message // @start // @next EthereumMessageSignature @@ -514,7 +514,7 @@ func (m *EthereumSignMessage) GetMessage() []byte { return nil } -//* +// * // Response: Signed message // @end type EthereumMessageSignature struct { @@ -572,7 +572,7 @@ func (m *EthereumMessageSignature) GetAddressHex() string { return "" } -//* +// * // Request: Ask device to verify message // @start // @next Success diff --git a/accounts/usbwallet/trezor/messages-management.pb.go b/accounts/usbwallet/trezor/messages-management.pb.go index f5c872f1fb5ba..91bfca1e3f086 100644 --- a/accounts/usbwallet/trezor/messages-management.pb.go +++ b/accounts/usbwallet/trezor/messages-management.pb.go @@ -21,7 +21,7 @@ var _ = math.Inf // proto package needs to be updated. const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package -//* +// * // Structure representing passphrase source type ApplySettings_PassphraseSourceType int32 @@ -66,7 +66,7 @@ func (ApplySettings_PassphraseSourceType) EnumDescriptor() ([]byte, []int) { return fileDescriptor_0c720c20d27aa029, []int{4, 0} } -//* +// * // Type of recovery procedure. These should be used as bitmask, e.g., // `RecoveryDeviceType_ScrambledWords | RecoveryDeviceType_Matrix` // listing every method supported by the host computer. @@ -114,7 +114,7 @@ func (RecoveryDevice_RecoveryDeviceType) EnumDescriptor() ([]byte, []int) { return fileDescriptor_0c720c20d27aa029, []int{17, 0} } -//* +// * // Type of Recovery Word request type WordRequest_WordRequestType int32 @@ -159,7 +159,7 @@ func (WordRequest_WordRequestType) EnumDescriptor() ([]byte, []int) { return fileDescriptor_0c720c20d27aa029, []int{18, 0} } -//* +// * // Request: Reset device to default state and ask for device details // @start // @next Features @@ -210,7 +210,7 @@ func (m *Initialize) GetSkipPassphrase() bool { return false } -//* +// * // Request: Ask for device details (no device reset) // @start // @next Features @@ -245,7 +245,7 @@ func (m *GetFeatures) XXX_DiscardUnknown() { var xxx_messageInfo_GetFeatures proto.InternalMessageInfo -//* +// * // Response: Reports various information about the device // @end type Features struct { @@ -495,7 +495,7 @@ func (m *Features) GetNoBackup() bool { return false } -//* +// * // Request: clear session (removes cached PIN, passphrase, etc). // @start // @next Success @@ -530,7 +530,7 @@ func (m *ClearSession) XXX_DiscardUnknown() { var xxx_messageInfo_ClearSession proto.InternalMessageInfo -//* +// * // Request: change language and/or label of the device // @start // @next Success @@ -622,7 +622,7 @@ func (m *ApplySettings) GetDisplayRotation() uint32 { return 0 } -//* +// * // Request: set flags of the device // @start // @next Success @@ -666,7 +666,7 @@ func (m *ApplyFlags) GetFlags() uint32 { return 0 } -//* +// * // Request: Starts workflow for setting/changing/removing the PIN // @start // @next Success @@ -710,7 +710,7 @@ func (m *ChangePin) GetRemove() bool { return false } -//* +// * // Request: Test if the device is alive, device sends back the message in Success response // @start // @next Success @@ -777,7 +777,7 @@ func (m *Ping) GetPassphraseProtection() bool { return false } -//* +// * // Request: Abort last operation that required user interaction // @start // @next Failure @@ -812,7 +812,7 @@ func (m *Cancel) XXX_DiscardUnknown() { var xxx_messageInfo_Cancel proto.InternalMessageInfo -//* +// * // Request: Request a sample of random data generated by hardware RNG. May be used for testing. // @start // @next Entropy @@ -856,7 +856,7 @@ func (m *GetEntropy) GetSize() uint32 { return 0 } -//* +// * // Response: Reply with random data generated by internal RNG // @end type Entropy struct { @@ -898,7 +898,7 @@ func (m *Entropy) GetEntropy() []byte { return nil } -//* +// * // Request: Request device to wipe all sensitive data and settings // @start // @next Success @@ -934,7 +934,7 @@ func (m *WipeDevice) XXX_DiscardUnknown() { var xxx_messageInfo_WipeDevice proto.InternalMessageInfo -//* +// * // Request: Load seed and related internal settings from the computer // @start // @next Success @@ -1036,7 +1036,7 @@ func (m *LoadDevice) GetU2FCounter() uint32 { return 0 } -//* +// * // Request: Ask device to do initialization involving user interaction // @start // @next EntropyRequest @@ -1147,7 +1147,7 @@ func (m *ResetDevice) GetNoBackup() bool { return false } -//* +// * // Request: Perform backup of the device seed if not backed up using ResetDevice // @start // @next Success @@ -1182,7 +1182,7 @@ func (m *BackupDevice) XXX_DiscardUnknown() { var xxx_messageInfo_BackupDevice proto.InternalMessageInfo -//* +// * // Response: Ask for additional entropy from host computer // @next EntropyAck type EntropyRequest struct { @@ -1216,7 +1216,7 @@ func (m *EntropyRequest) XXX_DiscardUnknown() { var xxx_messageInfo_EntropyRequest proto.InternalMessageInfo -//* +// * // Request: Provide additional entropy for seed generation function // @next Success type EntropyAck struct { @@ -1258,7 +1258,7 @@ func (m *EntropyAck) GetEntropy() []byte { return nil } -//* +// * // Request: Start recovery workflow asking user for specific words of mnemonic // Used to recovery device safely even on untrusted computer. // @start @@ -1369,7 +1369,7 @@ func (m *RecoveryDevice) GetDryRun() bool { return false } -//* +// * // Response: Device is waiting for user to enter word of the mnemonic // Its position is shown only on device's internal display. // @next WordAck @@ -1412,7 +1412,7 @@ func (m *WordRequest) GetType() WordRequest_WordRequestType { return WordRequest_WordRequestType_Plain } -//* +// * // Request: Computer replies with word from the mnemonic // @next WordRequest // @next Success @@ -1456,7 +1456,7 @@ func (m *WordAck) GetWord() string { return "" } -//* +// * // Request: Set U2F counter // @start // @next Success diff --git a/accounts/usbwallet/trezor/messages.pb.go b/accounts/usbwallet/trezor/messages.pb.go index 6278bd8ee02cd..af0c957144d23 100644 --- a/accounts/usbwallet/trezor/messages.pb.go +++ b/accounts/usbwallet/trezor/messages.pb.go @@ -22,7 +22,7 @@ var _ = math.Inf // proto package needs to be updated. const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package -//* +// * // Mapping between TREZOR wire identifier (uint) and a protobuf message type MessageType int32 diff --git a/accounts/usbwallet/wallet.go b/accounts/usbwallet/wallet.go index 382f3ddaee217..0e399a6d09abb 100644 --- a/accounts/usbwallet/wallet.go +++ b/accounts/usbwallet/wallet.go @@ -380,7 +380,7 @@ func (w *wallet) selfDerive() { // of legacy-ledger, the first account on the legacy-path will // be shown to the user, even if we don't actively track it if i < len(nextAddrs)-1 { - w.log.Info("Skipping trakcking first account on legacy path, use personal.deriveAccount(,, false) to track", + w.log.Info("Skipping tracking first account on legacy path, use personal.deriveAccount(,, false) to track", "path", path, "address", nextAddrs[i]) break } @@ -526,7 +526,6 @@ func (w *wallet) signHash(account accounts.Account, hash []byte) ([]byte, error) // SignData signs keccak256(data). The mimetype parameter describes the type of data being signed func (w *wallet) SignData(account accounts.Account, mimeType string, data []byte) ([]byte, error) { - // Unless we are doing 712 signing, simply dispatch to signHash if !(mimeType == accounts.MimetypeTypedData && len(data) == 66 && data[0] == 0x19 && data[1] == 0x01) { return w.signHash(account, crypto.Keccak256(data)) diff --git a/build/checksums.txt b/build/checksums.txt index 4d6176ecbe59b..2725329fbc090 100644 --- a/build/checksums.txt +++ b/build/checksums.txt @@ -1,58 +1,38 @@ # This file contains sha256 checksums of optional build dependencies. -efd43e0f1402e083b73a03d444b7b6576bb4c539ac46208b63a916b69aca4088 go1.18.1.src.tar.gz -3703e9a0db1000f18c0c7b524f3d378aac71219b4715a6a4c5683eb639f41a4d go1.18.1.darwin-amd64.tar.gz -6d5641a06edba8cd6d425fb0adad06bad80e2afe0fa91b4aa0e5aed1bc78f58e go1.18.1.darwin-arm64.tar.gz -b9a9063d4265d8ccc046c9b314194d6eadc47e56d0d637db81e98e68aad45035 go1.18.1.freebsd-386.tar.gz -2bc1c138d645e37dbbc63517dd1cf1bf33fc4cb95f442a6384df0418b5134e9f go1.18.1.freebsd-amd64.tar.gz -9a8df5dde9058f08ac01ecfaae42534610db398e487138788c01da26a0d41ff9 go1.18.1.linux-386.tar.gz -b3b815f47ababac13810fc6021eb73d65478e0b2db4b09d348eefad9581a2334 go1.18.1.linux-amd64.tar.gz -56a91851c97fb4697077abbca38860f735c32b38993ff79b088dac46e4735633 go1.18.1.linux-arm64.tar.gz -9edc01c8e7db64e9ceeffc8258359e027812886ceca3444e83c4eb96ddb068ee go1.18.1.linux-armv6l.tar.gz -33db623d1eecf362fe365107c12efc90eff0b9609e0b3345e258388019cb552a go1.18.1.linux-ppc64le.tar.gz -5d9301324148ed4dbfaa0800da43a843ffd65c834ee73fcf087255697c925f74 go1.18.1.linux-s390x.tar.gz -49ae65551acbfaa57b52fbefa0350b2072512ae3103b8cf1a919a02626dbc743 go1.18.1.windows-386.zip -c30bc3f1f7314a953fe208bd9cd5e24bd9403392a6c556ced3677f9f70f71fe1 go1.18.1.windows-amd64.zip -2c4a8265030eac37f906634f5c13c22c3d0ea725f2488e1bca005c6b981653d7 go1.18.1.windows-arm64.zip +27871baa490f3401414ad793fba49086f6c855b1c584385ed7771e1204c7e179 go1.19.1.src.tar.gz +b2828a2b05f0d2169afc74c11ed010775bf7cf0061822b275697b2f470495fb7 go1.19.1.darwin-amd64.tar.gz +e46aecce83a9289be16ce4ba9b8478a5b89b8aa0230171d5c6adbc0c66640548 go1.19.1.darwin-arm64.tar.gz +cfaca8c1d5784d2bc21e12d8893cfd2dc885a60db4c1a9a95e4ffc694d0925ce go1.19.1.freebsd-386.tar.gz +db5b8f232e12c655cc6cde6af1adf4d27d842541807802d747c86161e89efa0a go1.19.1.freebsd-amd64.tar.gz +9acc57342400c5b0c2da07b5b01b50da239dd4a7fad41a1fb56af8363ef4133f go1.19.1.linux-386.tar.gz +acc512fbab4f716a8f97a8b3fbaa9ddd39606a28be6c2515ef7c6c6311acffde go1.19.1.linux-amd64.tar.gz +49960821948b9c6b14041430890eccee58c76b52e2dbaafce971c3c38d43df9f go1.19.1.linux-arm64.tar.gz +efe93f5671621ee84ce5e262e1e21acbc72acefbaba360f21778abd083d4ad16 go1.19.1.linux-armv6l.tar.gz +4137984aa353de9c5ec1bd8fb3cd00a0624b75eafa3d4ec13d2f3f48261dba2e go1.19.1.linux-ppc64le.tar.gz +ca1005cc80a3dd726536b4c6ea5fef0318939351ff273eff420bd66a377c74eb go1.19.1.linux-s390x.tar.gz +bc7043e7a9a8d34aacd06f8c2f70e166d1d148f6800814cff790c45b9ab31cee go1.19.1.windows-386.zip +b33584c1d93b0e9c783de876b7aa99d3018bdeccd396aeb6d516a74e9d88d55f go1.19.1.windows-amd64.zip +d8cf3f04762fa7d5d9c82dfa15b5adaae2404463af3bc8dcd7f89837512501fe go1.19.1.windows-arm64.zip -03c181fc1bb29ea3e73cbb23399c43b081063833a7cf7554b94e5a98308df53e golangci-lint-1.45.2-linux-riscv64.deb -08a50bbbf451ede6d5354179eb3e14a5634e156dfa92cb9a2606f855a637e35b golangci-lint-1.45.2-linux-ppc64le.rpm -0d12f6ec1296b5a70e392aa88cd2295cceef266165eb7028e675f455515dd1c9 golangci-lint-1.45.2-linux-armv7.deb -10f2846e2e50e4ea8ae426ee62dcd2227b23adddd8e991aa3c065927ac948735 golangci-lint-1.45.2-linux-ppc64le.deb -1463049b744871168095e3e8f687247d6040eeb895955b869889ea151e0603ab golangci-lint-1.45.2-linux-arm64.tar.gz -15720f9c4c6f9324af695f081dc189adc7751b255759e78d7b2df1d7e9192533 golangci-lint-1.45.2-linux-amd64.deb -166d922e4d3cfe3d47786c590154a9c8ea689dff0aa92b73d2f5fc74fc570c29 golangci-lint-1.45.2-linux-arm64.rpm -1a3754c69f7cc19ab89cbdcc2550da4cf9abb3120383c6b3bd440c1ec22da2e6 golangci-lint-1.45.2-freebsd-386.tar.gz -1dec0aa46d4f0d241863b573f70129bdf1de9c595cf51172a840a588a4cd9fc5 golangci-lint-1.45.2-windows-amd64.zip -3198453806517c1ad988229f5e758ef850e671203f46d6905509df5bdf4dc24b golangci-lint-1.45.2-freebsd-armv7.tar.gz -46a3cd1749d7b98adc2dc01510ddbe21abe42689c8a53fb0e81662713629f215 golangci-lint-1.45.2-linux-386.deb -4e28bfb593d464b9e160f2acd5b71993836a183270bf8299b78ad31f7a168c0d golangci-lint-1.45.2-linux-arm64.deb -5157a58c8f9ab85c33af2e46f0d7c57a3b1e8953b81d61130e292e09f545cfab golangci-lint-1.45.2-linux-mips64le.tar.gz -518cd027644129fbf8ec4f02bd6f9ad7278aae826f92b63c80d4d0819ddde49a golangci-lint-1.45.2-linux-armv6.rpm -595ad6c6dade4c064351bc309f411703e457f8ffbb7a1806b3d8ee713333427f golangci-lint-1.45.2-linux-amd64.tar.gz -6994d6c80f0730751090986184a3481b4be2e6b6e84416238a2b857910045a4f golangci-lint-1.45.2-windows-arm64.zip -6c81652fc340118811b487f713c441fc6f527800bf5fd11b8929d08124efa015 golangci-lint-1.45.2-linux-armv7.tar.gz -726cb045559b7518bafdd3459de70a0647c087eb1b4634627a4b2e95b1258580 golangci-lint-1.45.2-freebsd-amd64.tar.gz -77df3774cdfda49b956d4a0e676da9a9b883f496ee37293c530770fef6b1d24e golangci-lint-1.45.2-linux-mips64.deb -7a9840f279a7d5d405bb434e101c2290964b3729630ac2add29280b962b7b9a5 golangci-lint-1.45.2-windows-armv6.zip -7d4bf9a5d80ec467aaaf66e78dbdcab567bbc6ba8151334c714eee58766aae32 golangci-lint-1.45.2-windows-armv7.zip -7e5f8821d39bb11d273b0841b34355f56bd5a45a2d5179f0d09e614e0efc0482 golangci-lint-1.45.2-linux-s390x.rpm -828de1bde796b23d8656b17a8885fbd879ef612795d62d1e4618126b419728b5 golangci-lint-1.45.2-linux-mips64.rpm -879a52107a797678a03c175cc7cf441411a14a01f66dc87f70bdfa304a4129a6 golangci-lint-1.45.2-windows-386.zip -87b6c7e3a3769f7d9abeb3bb82119b3c91e3c975300f6834fdeef8b2e37c98ff golangci-lint-1.45.2-linux-amd64.rpm -8b605c6d686c8af53ecc4ef39544541eeb1644d34cc10f9ffc5087808210c4ff golangci-lint-1.45.2-linux-s390x.deb -9427dbf51d0ac6f73a0f992838bd40c817470cc5bf6c8e2e2bea6fac46d7af6e golangci-lint-1.45.2-linux-ppc64le.tar.gz -995e509e895ca6a64ffc7395ac884d5961bdec98423cb896b17f345a9b4a19cf golangci-lint-1.45.2-darwin-amd64.tar.gz -a3f36278f2ea5516341e9071a2df6e65df272be80230b5406a12b72c6d425bee golangci-lint-1.45.2-linux-armv7.rpm -a5e12c50c23e87ac1deffc872f92ae85427b1198604969399805ae47cfe43f08 golangci-lint-1.45.2-linux-riscv64.tar.gz -aa8fa1be0729dbc2fbc4e01e82027097613eee74bd686ebef20f860b01fff8b3 golangci-lint-1.45.2-freebsd-armv6.tar.gz -c2b9669decc1b638cf2ee9060571af4e255f6dfcbb225c293e3a7ee4bb2c7217 golangci-lint-1.45.2-darwin-arm64.tar.gz -dfa8bdaf0387aec1cd5c1aa8857f67b2bbdfc2e42efce540c8fb9bbe3e8af302 golangci-lint-1.45.2-linux-armv6.tar.gz -eb8b8539dd017eee5c131ea9b875893ab2cebeeca41e8c6624907fb02224d643 golangci-lint-1.45.2-linux-386.rpm -ed6c7e17a857f30d715c5302fa250d95936936b277024bffea201187a257d7a7 golangci-lint-1.45.2-linux-armv6.deb -ef4d0154ace4001f01b288baeb118176242efb4fd163e178763e3213b77ef30b golangci-lint-1.45.2-linux-mips64le.deb -ef7002a2229f5ff5ba201a715fcf877664ea88decbe58e69d163293913024955 golangci-lint-1.45.2-linux-s390x.tar.gz -f13ecbd09228632e6bbe91a8324bd675c406eed22eb6d2c1e8192eed9ec4f914 golangci-lint-1.45.2-linux-386.tar.gz -f4cd9cfb09252f51699407277512263cae8409b665dd764f55a34738d0e89edc golangci-lint-1.45.2-linux-riscv64.rpm -fb1945dc59d37c9d14bf0a4aea11ea8651fa0e1d582ea80c4c44d0a536c08893 golangci-lint-1.45.2-linux-mips64.tar.gz -fe542c22738010f453c735a3c410decfd3784d1bd394b395c298ee298fc4c606 golangci-lint-1.45.2-linux-mips64le.rpm +20cd1215e0420db8cfa94a6cd3c9d325f7b39c07f2415a02d111568d8bc9e271 golangci-lint-1.49.0-darwin-amd64.tar.gz +cabb1a4c35fe1dadbe5a81550a00871281a331e7660cd85ae16e936a7f0f6cfc golangci-lint-1.49.0-darwin-arm64.tar.gz +f834c3b09580cf763b5d30b0c33c83cb13d7a822b5ed5d724143f121ffe28c97 golangci-lint-1.49.0-freebsd-386.tar.gz +4ca91c9f3aa79a71da441b7220a3e799365ff7a24caf9f04fcda12066c5ab0f7 golangci-lint-1.49.0-freebsd-amd64.tar.gz +37de789245248eea375d05080e11b4662a08762c353752575167611e65658454 golangci-lint-1.49.0-freebsd-armv6.tar.gz +3abed2bd3a8134b501fdc9cc9a0e60d616c86389e4fcdd1f79ceae7458974378 golangci-lint-1.49.0-freebsd-armv7.tar.gz +ef2860d90d83aee6713f697f23372cd93ac41a16439fdcb3c4ac86ba0f306860 golangci-lint-1.49.0-linux-386.tar.gz +5badc6e9fee2003621efa07e385910d9a88c89b38f6c35aded153193c5125178 golangci-lint-1.49.0-linux-amd64.tar.gz +b57ed03d29b8ca69be9925edd67ea305b6013cd5c97507d205fbe2979f71f2b5 golangci-lint-1.49.0-linux-arm64.tar.gz +4a41cff3af7f5304751f7bbf4ea617c14ebc1f88481a28a013e61b06d1f7102c golangci-lint-1.49.0-linux-armv6.tar.gz +14a9683af483ee7052dd0ce7d6140e0b502d6001bea3de606b8e7cce2c673539 golangci-lint-1.49.0-linux-armv7.tar.gz +33edf757bc2611204fdb40b212900866a57ded4eea62c1b19c10bfc375359afa golangci-lint-1.49.0-linux-mips64.tar.gz +280f7902f90d162566f1691a300663dd8db6e225e65384fe66b6fb2362e0b314 golangci-lint-1.49.0-linux-mips64le.tar.gz +103bcb7ce6c668e0a7e95e5c5355892d74f5d15391443430472e66d652906a15 golangci-lint-1.49.0-linux-ppc64le.tar.gz +4636ff9b01ddb18a2c1a953fc134207711b0a5d874d04ac66f915e9cfff0e8e0 golangci-lint-1.49.0-linux-riscv64.tar.gz +029e0844931a2d3edc771d67e17fe17928f04f80c1a9aa165160a543e8a7e8d4 golangci-lint-1.49.0-linux-s390x.tar.gz +e9cb6f691e62a4d8b28dd52d2eab57cca72acfd5083b3c5417a72d2eb64def09 golangci-lint-1.49.0-windows-386.zip +d058dfb0c7fbd73be70f285d3f8d4d424192fe9b19760ddbb0b2c4b743b8656c golangci-lint-1.49.0-windows-amd64.zip +c049d80297228db7065eabeac5114f77f04415dcd9b944e8d7c6426d9dd6e9dd golangci-lint-1.49.0-windows-arm64.zip +ec9164bab7134ddb94f51c17fd37c109b0801ecd5494b6c0e27ca7898fbd7469 golangci-lint-1.49.0-windows-armv6.zip +68fd9e880d98073f436c58b6f6d2c141881ef49b06ca31137bc19da4e4e3b996 golangci-lint-1.49.0-windows-armv7.zip diff --git a/build/ci.go b/build/ci.go index 4129168be53a7..043c13b76e3c6 100644 --- a/build/ci.go +++ b/build/ci.go @@ -24,19 +24,18 @@ Usage: go run build/ci.go Available commands are: - install [ -arch architecture ] [ -cc compiler ] [ packages... ] -- builds packages and executables - test [ -coverage ] [ packages... ] -- runs the tests - lint -- runs certain pre-selected linters - archive [ -arch architecture ] [ -type zip|tar ] [ -signer key-envvar ] [ -signify key-envvar ] [ -upload dest ] -- archives build artifacts - importkeys -- imports signing keys from env - debsrc [ -signer key-id ] [ -upload dest ] -- creates a debian source package - nsis -- creates a Windows NSIS installer - aar [ -local ] [ -sign key-id ] [-deploy repo] [ -upload dest ] -- creates an Android archive - xcode [ -local ] [ -sign key-id ] [-deploy repo] [ -upload dest ] -- creates an iOS XCode framework - purge [ -store blobstore ] [ -days threshold ] -- purges old archives from the blobstore + install [ -arch architecture ] [ -cc compiler ] [ packages... ] -- builds packages and executables + test [ -coverage ] [ packages... ] -- runs the tests + lint -- runs certain pre-selected linters + archive [ -arch architecture ] [ -type zip|tar ] [ -signer key-envvar ] [ -signify key-envvar ] [ -upload dest ] -- archives build artifacts + importkeys -- imports signing keys from env + debsrc [ -signer key-id ] [ -upload dest ] -- creates a debian source package + nsis -- creates a Windows NSIS installer + aar [ -local ] [ -sign key-id ] [-deploy repo] [ -upload dest ] -- creates an Android archive + xcode [ -local ] [ -sign key-id ] [-deploy repo] [ -upload dest ] -- creates an iOS XCode framework + purge [ -store blobstore ] [ -days threshold ] -- purges old archives from the blobstore For all commands, -n prevents execution of external programs (dry run mode). - */ package main @@ -132,12 +131,12 @@ var ( // Note: the following Ubuntu releases have been officially deprecated on Launchpad: // wily, yakkety, zesty, artful, cosmic, disco, eoan, groovy, hirsuite debDistroGoBoots = map[string]string{ - "trusty": "golang-1.11", // EOL: 04/2024 - "xenial": "golang-go", // EOL: 04/2026 - "bionic": "golang-go", // EOL: 04/2028 - "focal": "golang-go", // EOL: 04/2030 - "impish": "golang-go", // EOL: 07/2022 - "jammy": "golang-go", // EOL: 04/2032 + "trusty": "golang-1.11", // EOL: 04/2024 + "xenial": "golang-go", // EOL: 04/2026 + "bionic": "golang-go", // EOL: 04/2028 + "focal": "golang-go", // EOL: 04/2030 + "impish": "golang-go", // EOL: 07/2022 + "jammy": "golang-go", // EOL: 04/2032 //"kinetic": "golang-go", // EOL: 07/2023 } @@ -149,7 +148,7 @@ var ( // This is the version of go that will be downloaded by // // go run ci.go install -dlgo - dlgoVersion = "1.18.1" + dlgoVersion = "1.19.1" ) var GOBIN, _ = filepath.Abs(filepath.Join("build", "bin")) @@ -200,9 +199,10 @@ func main() { func doInstall(cmdline []string) { var ( - dlgo = flag.Bool("dlgo", false, "Download Go and build with it") - arch = flag.String("arch", "", "Architecture to cross build for") - cc = flag.String("cc", "", "C compiler to cross build with") + dlgo = flag.Bool("dlgo", false, "Download Go and build with it") + arch = flag.String("arch", "", "Architecture to cross build for") + cc = flag.String("cc", "", "C compiler to cross build with") + staticlink = flag.Bool("static", false, "Create statically-linked executable") ) flag.CommandLine.Parse(cmdline) @@ -213,9 +213,12 @@ func doInstall(cmdline []string) { tc.Root = build.DownloadGo(csdb, dlgoVersion) } + // Disable CLI markdown doc generation in release builds. + buildTags := []string{"urfave_cli_no_docs"} + // Configure the build. env := build.Env() - gobuild := tc.Go("build", buildFlags(env)...) + gobuild := tc.Go("build", buildFlags(env, *staticlink, buildTags)...) // arm64 CI builders are memory-constrained and can't handle concurrent builds, // better disable it. This check isn't the best, it should probably @@ -248,25 +251,35 @@ func doInstall(cmdline []string) { } // buildFlags returns the go tool flags for building. -func buildFlags(env build.Environment) (flags []string) { +func buildFlags(env build.Environment, staticLinking bool, buildTags []string) (flags []string) { var ld []string if env.Commit != "" { - ld = append(ld, "-X", "main.gitCommit="+env.Commit) - ld = append(ld, "-X", "main.gitDate="+env.Date) + ld = append(ld, "-X", "github.com/ethereum/go-ethereum/internal/version.gitCommit="+env.Commit) + ld = append(ld, "-X", "github.com/ethereum/go-ethereum/internal/version.gitDate="+env.Date) } // Strip DWARF on darwin. This used to be required for certain things, // and there is no downside to this, so we just keep doing it. if runtime.GOOS == "darwin" { ld = append(ld, "-s") } - // Enforce the stacksize to 8M, which is the case on most platforms apart from - // alpine Linux. if runtime.GOOS == "linux" { - ld = append(ld, "-extldflags", "-Wl,-z,stack-size=0x800000") + // Enforce the stacksize to 8M, which is the case on most platforms apart from + // alpine Linux. + extld := []string{"-Wl,-z,stack-size=0x800000"} + if staticLinking { + extld = append(extld, "-static") + // Under static linking, use of certain glibc features must be + // disabled to avoid shared library dependencies. + buildTags = append(buildTags, "osusergo", "netgo") + } + ld = append(ld, "-extldflags", "'"+strings.Join(extld, " ")+"'") } if len(ld) > 0 { flags = append(flags, "-ldflags", strings.Join(ld, " ")) } + if len(buildTags) > 0 { + flags = append(flags, "-tags", strings.Join(buildTags, ",")) + } return flags } @@ -333,7 +346,7 @@ func doLint(cmdline []string) { // downloadLinter downloads and unpacks golangci-lint. func downloadLinter(cachedir string) string { - const version = "1.45.2" + const version = "1.49.0" csdb := build.MustLoadChecksums("build/checksums.txt") arch := runtime.GOARCH @@ -594,7 +607,7 @@ func doDocker(cmdline []string) { } if mismatch { // Build numbers mismatching, retry in a short time to - // avoid concurrent failes in both publisher images. If + // avoid concurrent fails in both publisher images. If // however the retry failed too, it means the concurrent // builder is still crunching, let that do the publish. if i == 0 { @@ -968,7 +981,10 @@ func doWindowsInstaller(cmdline []string) { if env.Commit != "" { version[2] += "-" + env.Commit[:8] } - installer, _ := filepath.Abs("geth-" + archiveBasename(*arch, params.ArchiveVersion(env.Commit)) + ".exe") + installer, err := filepath.Abs("geth-" + archiveBasename(*arch, params.ArchiveVersion(env.Commit)) + ".exe") + if err != nil { + log.Fatalf("Failed to convert installer file path: %v", err) + } build.MustRunCommand("makensis.exe", "/DOUTPUTFILE="+installer, "/DMAJORVERSION="+version[0], diff --git a/build/deb/ethereum/completions/bash/geth b/build/deb/ethereum/completions/bash/geth new file mode 100755 index 0000000000000..a78952793efb1 --- /dev/null +++ b/build/deb/ethereum/completions/bash/geth @@ -0,0 +1,16 @@ +_geth_bash_autocomplete() { + if [[ "${COMP_WORDS[0]}" != "source" ]]; then + local cur opts base + COMPREPLY=() + cur="${COMP_WORDS[COMP_CWORD]}" + if [[ "$cur" == "-"* ]]; then + opts=$( ${COMP_WORDS[@]:0:$COMP_CWORD} ${cur} --generate-bash-completion ) + else + opts=$( ${COMP_WORDS[@]:0:$COMP_CWORD} --generate-bash-completion ) + fi + COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) + return 0 + fi +} + +complete -o bashdefault -o default -o nospace -F _geth_bash_autocomplete geth diff --git a/build/deb/ethereum/completions/zsh/_geth b/build/deb/ethereum/completions/zsh/_geth new file mode 100644 index 0000000000000..119794c532bda --- /dev/null +++ b/build/deb/ethereum/completions/zsh/_geth @@ -0,0 +1,18 @@ +_geth_zsh_autocomplete() { + local -a opts + local cur + cur=${words[-1]} + if [[ "$cur" == "-"* ]]; then + opts=("${(@f)$(${words[@]:0:#words[@]-1} ${cur} --generate-bash-completion)}") + else + opts=("${(@f)$(${words[@]:0:#words[@]-1} --generate-bash-completion)}") + fi + + if [[ "${opts[1]}" != "" ]]; then + _describe 'values' opts + else + _files + fi +} + +compdef _geth_zsh_autocomplete geth diff --git a/build/deb/ethereum/deb.install b/build/deb/ethereum/deb.install index e7666ce5fb6b4..5a624594b06ca 100644 --- a/build/deb/ethereum/deb.install +++ b/build/deb/ethereum/deb.install @@ -1 +1,5 @@ build/bin/{{.BinaryName}} usr/bin +{{- if eq .BinaryName "geth" }} +build/deb/ethereum/completions/bash/geth etc/bash_completion.d +build/deb/ethereum/completions/zsh/_geth usr/share/zsh/vendor-completions +{{end -}} diff --git a/build/update-license.go b/build/update-license.go index 5bad996cc45b3..f61536470a195 100644 --- a/build/update-license.go +++ b/build/update-license.go @@ -342,7 +342,10 @@ func isGenerated(file string) bool { } defer fd.Close() buf := make([]byte, 2048) - n, _ := fd.Read(buf) + n, err := fd.Read(buf) + if err != nil { + return false + } buf = buf[:n] for _, l := range bytes.Split(buf, []byte("\n")) { if bytes.HasPrefix(l, []byte("// Code generated")) { diff --git a/cmd/abigen/main.go b/cmd/abigen/main.go index 7a321e18b6b5b..83b6c5e4289f2 100644 --- a/cmd/abigen/main.go +++ b/cmd/abigen/main.go @@ -30,58 +30,54 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/internal/flags" "github.com/ethereum/go-ethereum/log" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) var ( - // Git SHA1 commit hash of the release (set via linker flags) - gitCommit = "" - gitDate = "" - - app *cli.App - // Flags needed by abigen - abiFlag = cli.StringFlag{ + abiFlag = &cli.StringFlag{ Name: "abi", Usage: "Path to the Ethereum contract ABI json to bind, - for STDIN", } - binFlag = cli.StringFlag{ + binFlag = &cli.StringFlag{ Name: "bin", Usage: "Path to the Ethereum contract bytecode (generate deploy method)", } - typeFlag = cli.StringFlag{ + typeFlag = &cli.StringFlag{ Name: "type", Usage: "Struct name for the binding (default = package name)", } - jsonFlag = cli.StringFlag{ + jsonFlag = &cli.StringFlag{ Name: "combined-json", Usage: "Path to the combined-json file generated by compiler, - for STDIN", } - excFlag = cli.StringFlag{ + excFlag = &cli.StringFlag{ Name: "exc", Usage: "Comma separated types to exclude from binding", } - pkgFlag = cli.StringFlag{ + pkgFlag = &cli.StringFlag{ Name: "pkg", Usage: "Package name to generate the binding into", } - outFlag = cli.StringFlag{ + outFlag = &cli.StringFlag{ Name: "out", Usage: "Output file for the generated binding (default = stdout)", } - langFlag = cli.StringFlag{ + langFlag = &cli.StringFlag{ Name: "lang", Usage: "Destination language for the bindings (go, java, objc)", Value: "go", } - aliasFlag = cli.StringFlag{ + aliasFlag = &cli.StringFlag{ Name: "alias", Usage: "Comma separated aliases for function and event renaming, e.g. original1=alias1, original2=alias2", } ) +var app = flags.NewApp("Ethereum ABI wrapper code generator") + func init() { - app = flags.NewApp(gitCommit, gitDate, "ethereum checkpoint helper tool") + app.Name = "abigen" app.Flags = []cli.Flag{ abiFlag, binFlag, @@ -93,17 +89,17 @@ func init() { langFlag, aliasFlag, } - app.Action = utils.MigrateFlags(abigen) - cli.CommandHelpTemplate = flags.OriginCommandHelpTemplate + app.Action = abigen } func abigen(c *cli.Context) error { utils.CheckExclusive(c, abiFlag, jsonFlag) // Only one source can be selected. - if c.GlobalString(pkgFlag.Name) == "" { + + if c.String(pkgFlag.Name) == "" { utils.Fatalf("No destination package specified (--pkg)") } var lang bind.Lang - switch c.GlobalString(langFlag.Name) { + switch c.String(langFlag.Name) { case "go": lang = bind.LangGo case "java": @@ -112,7 +108,7 @@ func abigen(c *cli.Context) error { lang = bind.LangObjC utils.Fatalf("Objc binding generation is uncompleted") default: - utils.Fatalf("Unsupported destination language \"%s\" (--lang)", c.GlobalString(langFlag.Name)) + utils.Fatalf("Unsupported destination language \"%s\" (--lang)", c.String(langFlag.Name)) } // If the entire solidity code was specified, build and bind based on that var ( @@ -123,13 +119,13 @@ func abigen(c *cli.Context) error { libs = make(map[string]string) aliases = make(map[string]string) ) - if c.GlobalString(abiFlag.Name) != "" { + if c.String(abiFlag.Name) != "" { // Load up the ABI, optional bytecode and type name from the parameters var ( abi []byte err error ) - input := c.GlobalString(abiFlag.Name) + input := c.String(abiFlag.Name) if input == "-" { abi, err = io.ReadAll(os.Stdin) } else { @@ -141,7 +137,7 @@ func abigen(c *cli.Context) error { abis = append(abis, string(abi)) var bin []byte - if binFile := c.GlobalString(binFlag.Name); binFile != "" { + if binFile := c.String(binFlag.Name); binFile != "" { if bin, err = os.ReadFile(binFile); err != nil { utils.Fatalf("Failed to read input bytecode: %v", err) } @@ -151,22 +147,25 @@ func abigen(c *cli.Context) error { } bins = append(bins, string(bin)) - kind := c.GlobalString(typeFlag.Name) + kind := c.String(typeFlag.Name) if kind == "" { - kind = c.GlobalString(pkgFlag.Name) + kind = c.String(pkgFlag.Name) } types = append(types, kind) } else { // Generate the list of types to exclude from binding - exclude := make(map[string]bool) - for _, kind := range strings.Split(c.GlobalString(excFlag.Name), ",") { - exclude[strings.ToLower(kind)] = true + var exclude *nameFilter + if c.IsSet(excFlag.Name) { + var err error + if exclude, err = newNameFilter(strings.Split(c.String(excFlag.Name), ",")...); err != nil { + utils.Fatalf("Failed to parse excludes: %v", err) + } } var contracts map[string]*compiler.Contract - if c.GlobalIsSet(jsonFlag.Name) { + if c.IsSet(jsonFlag.Name) { var ( - input = c.GlobalString(jsonFlag.Name) + input = c.String(jsonFlag.Name) jsonOutput []byte err error ) @@ -185,7 +184,11 @@ func abigen(c *cli.Context) error { } // Gather all non-excluded contract for binding for name, contract := range contracts { - if exclude[strings.ToLower(name)] { + // fully qualified name is of the form : + nameParts := strings.Split(name, ":") + typeName := nameParts[len(nameParts)-1] + if exclude != nil && exclude.Matches(name) { + fmt.Fprintf(os.Stderr, "excluding: %v\n", name) continue } abi, err := json.Marshal(contract.Info.AbiDefinition) // Flatten the compiler parse @@ -195,36 +198,39 @@ func abigen(c *cli.Context) error { abis = append(abis, string(abi)) bins = append(bins, contract.Code) sigs = append(sigs, contract.Hashes) - nameParts := strings.Split(name, ":") - types = append(types, nameParts[len(nameParts)-1]) + types = append(types, typeName) - libPattern := crypto.Keccak256Hash([]byte(name)).String()[2:36] - libs[libPattern] = nameParts[len(nameParts)-1] + // Derive the library placeholder which is a 34 character prefix of the + // hex encoding of the keccak256 hash of the fully qualified library name. + // Note that the fully qualified library name is the path of its source + // file and the library name separated by ":". + libPattern := crypto.Keccak256Hash([]byte(name)).String()[2:36] // the first 2 chars are 0x + libs[libPattern] = typeName } } // Extract all aliases from the flags - if c.GlobalIsSet(aliasFlag.Name) { + if c.IsSet(aliasFlag.Name) { // We support multi-versions for aliasing // e.g. // foo=bar,foo2=bar2 // foo:bar,foo2:bar2 re := regexp.MustCompile(`(?:(\w+)[:=](\w+))`) - submatches := re.FindAllStringSubmatch(c.GlobalString(aliasFlag.Name), -1) + submatches := re.FindAllStringSubmatch(c.String(aliasFlag.Name), -1) for _, match := range submatches { aliases[match[1]] = match[2] } } // Generate the contract binding - code, err := bind.Bind(types, abis, bins, sigs, c.GlobalString(pkgFlag.Name), lang, libs, aliases) + code, err := bind.Bind(types, abis, bins, sigs, c.String(pkgFlag.Name), lang, libs, aliases) if err != nil { utils.Fatalf("Failed to generate ABI binding: %v", err) } // Either flush it out to a file or display on the standard output - if !c.GlobalIsSet(outFlag.Name) { + if !c.IsSet(outFlag.Name) { fmt.Printf("%s\n", code) return nil } - if err := os.WriteFile(c.GlobalString(outFlag.Name), []byte(code), 0600); err != nil { + if err := os.WriteFile(c.String(outFlag.Name), []byte(code), 0600); err != nil { utils.Fatalf("Failed to write ABI binding: %v", err) } return nil diff --git a/cmd/abigen/namefilter.go b/cmd/abigen/namefilter.go new file mode 100644 index 0000000000000..eea5c643c4429 --- /dev/null +++ b/cmd/abigen/namefilter.go @@ -0,0 +1,58 @@ +package main + +import ( + "fmt" + "strings" +) + +type nameFilter struct { + fulls map[string]bool // path/to/contract.sol:Type + files map[string]bool // path/to/contract.sol:* + types map[string]bool // *:Type +} + +func newNameFilter(patterns ...string) (*nameFilter, error) { + f := &nameFilter{ + fulls: make(map[string]bool), + files: make(map[string]bool), + types: make(map[string]bool), + } + for _, pattern := range patterns { + if err := f.add(pattern); err != nil { + return nil, err + } + } + return f, nil +} + +func (f *nameFilter) add(pattern string) error { + ft := strings.Split(pattern, ":") + if len(ft) != 2 { + // filenames and types must not include ':' symbol + return fmt.Errorf("invalid pattern: %s", pattern) + } + + file, typ := ft[0], ft[1] + if file == "*" { + f.types[typ] = true + return nil + } else if typ == "*" { + f.files[file] = true + return nil + } + f.fulls[pattern] = true + return nil +} + +func (f *nameFilter) Matches(name string) bool { + ft := strings.Split(name, ":") + if len(ft) != 2 { + // If contract names are always of the fully-qualified form + // :, then this case will never happen. + return false + } + + file, typ := ft[0], ft[1] + // full paths > file paths > types + return f.fulls[name] || f.files[file] || f.types[typ] +} diff --git a/cmd/abigen/namefilter_test.go b/cmd/abigen/namefilter_test.go new file mode 100644 index 0000000000000..42ba55be5eb53 --- /dev/null +++ b/cmd/abigen/namefilter_test.go @@ -0,0 +1,38 @@ +package main + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNameFilter(t *testing.T) { + _, err := newNameFilter("Foo") + require.Error(t, err) + _, err = newNameFilter("too/many:colons:Foo") + require.Error(t, err) + + f, err := newNameFilter("a/path:A", "*:B", "c/path:*") + require.NoError(t, err) + + for _, tt := range []struct { + name string + match bool + }{ + {"a/path:A", true}, + {"unknown/path:A", false}, + {"a/path:X", false}, + {"unknown/path:X", false}, + {"any/path:B", true}, + {"c/path:X", true}, + {"c/path:foo:B", false}, + } { + match := f.Matches(tt.name) + if tt.match { + assert.True(t, match, "expected match") + } else { + assert.False(t, match, "expected no match") + } + } +} diff --git a/cmd/checkpoint-admin/common.go b/cmd/checkpoint-admin/common.go index 05a45dfbf9970..f86ac24f06c19 100644 --- a/cmd/checkpoint-admin/common.go +++ b/cmd/checkpoint-admin/common.go @@ -28,12 +28,12 @@ import ( "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) // newClient creates a client with specified remote URL. func newClient(ctx *cli.Context) *ethclient.Client { - client, err := ethclient.Dial(ctx.GlobalString(nodeURLFlag.Name)) + client, err := ethclient.Dial(ctx.String(nodeURLFlag.Name)) if err != nil { utils.Fatalf("Failed to connect to Ethereum node: %v", err) } @@ -64,9 +64,9 @@ func getContractAddr(client *rpc.Client) common.Address { func getCheckpoint(ctx *cli.Context, client *rpc.Client) *params.TrustedCheckpoint { var checkpoint *params.TrustedCheckpoint - if ctx.GlobalIsSet(indexFlag.Name) { + if ctx.IsSet(indexFlag.Name) { var result [3]string - index := uint64(ctx.GlobalInt64(indexFlag.Name)) + index := uint64(ctx.Int64(indexFlag.Name)) if err := client.Call(&result, "les_getCheckpoint", index); err != nil { utils.Fatalf("Failed to get local checkpoint %v, please ensure the les API is exposed", err) } diff --git a/cmd/checkpoint-admin/exec.go b/cmd/checkpoint-admin/exec.go index 352a96d9e6f00..cb67d0306d43a 100644 --- a/cmd/checkpoint-admin/exec.go +++ b/cmd/checkpoint-admin/exec.go @@ -36,10 +36,10 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) -var commandDeploy = cli.Command{ +var commandDeploy = &cli.Command{ Name: "deploy", Usage: "Deploy a new checkpoint oracle contract", Flags: []cli.Flag{ @@ -49,10 +49,10 @@ var commandDeploy = cli.Command{ signersFlag, thresholdFlag, }, - Action: utils.MigrateFlags(deploy), + Action: deploy, } -var commandSign = cli.Command{ +var commandSign = &cli.Command{ Name: "sign", Usage: "Sign the checkpoint with the specified key", Flags: []cli.Flag{ @@ -63,10 +63,10 @@ var commandSign = cli.Command{ hashFlag, oracleFlag, }, - Action: utils.MigrateFlags(sign), + Action: sign, } -var commandPublish = cli.Command{ +var commandPublish = &cli.Command{ Name: "publish", Usage: "Publish a checkpoint into the oracle", Flags: []cli.Flag{ @@ -76,7 +76,7 @@ var commandPublish = cli.Command{ indexFlag, signaturesFlag, }, - Action: utils.MigrateFlags(publish), + Action: publish, } // deploy deploys the checkpoint registrar contract. @@ -132,7 +132,7 @@ func sign(ctx *cli.Context) error { node *rpc.Client oracle *checkpointoracle.CheckpointOracle ) - if !ctx.GlobalIsSet(nodeURLFlag.Name) { + if !ctx.IsSet(nodeURLFlag.Name) { // Offline mode signing offline = true if !ctx.IsSet(hashFlag.Name) { @@ -151,7 +151,7 @@ func sign(ctx *cli.Context) error { address = common.HexToAddress(ctx.String(oracleFlag.Name)) } else { // Interactive mode signing, retrieve the data from the remote node - node = newRPCClient(ctx.GlobalString(nodeURLFlag.Name)) + node = newRPCClient(ctx.String(nodeURLFlag.Name)) checkpoint := getCheckpoint(ctx, node) chash, cindex, address = checkpoint.Hash(), checkpoint.SectionIndex, getContractAddr(node) @@ -265,7 +265,7 @@ func publish(ctx *cli.Context) error { } // Retrieve the checkpoint we want to sign to sort the signatures var ( - client = newRPCClient(ctx.GlobalString(nodeURLFlag.Name)) + client = newRPCClient(ctx.String(nodeURLFlag.Name)) addr, oracle = newContract(client) checkpoint = getCheckpoint(ctx, client) sighash = sighash(checkpoint.SectionIndex, addr, checkpoint.Hash()) diff --git a/cmd/checkpoint-admin/main.go b/cmd/checkpoint-admin/main.go index 0fb5532147787..ca0bae7375916 100644 --- a/cmd/checkpoint-admin/main.go +++ b/cmd/checkpoint-admin/main.go @@ -25,20 +25,13 @@ import ( "github.com/ethereum/go-ethereum/common/fdlimit" "github.com/ethereum/go-ethereum/internal/flags" "github.com/ethereum/go-ethereum/log" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) -var ( - // Git SHA1 commit hash of the release (set via linker flags) - gitCommit = "" - gitDate = "" -) - -var app *cli.App +var app = flags.NewApp("ethereum checkpoint helper tool") func init() { - app = flags.NewApp(gitCommit, gitDate, "ethereum checkpoint helper tool") - app.Commands = []cli.Command{ + app.Commands = []*cli.Command{ commandStatus, commandDeploy, commandSign, @@ -48,46 +41,45 @@ func init() { oracleFlag, nodeURLFlag, } - cli.CommandHelpTemplate = flags.OriginCommandHelpTemplate } // Commonly used command line flags. var ( - indexFlag = cli.Int64Flag{ + indexFlag = &cli.Int64Flag{ Name: "index", Usage: "Checkpoint index (query latest from remote node if not specified)", } - hashFlag = cli.StringFlag{ + hashFlag = &cli.StringFlag{ Name: "hash", Usage: "Checkpoint hash (query latest from remote node if not specified)", } - oracleFlag = cli.StringFlag{ + oracleFlag = &cli.StringFlag{ Name: "oracle", Usage: "Checkpoint oracle address (query from remote node if not specified)", } - thresholdFlag = cli.Int64Flag{ + thresholdFlag = &cli.Int64Flag{ Name: "threshold", Usage: "Minimal number of signatures required to approve a checkpoint", } - nodeURLFlag = cli.StringFlag{ + nodeURLFlag = &cli.StringFlag{ Name: "rpc", Value: "http://localhost:8545", Usage: "The rpc endpoint of a local or remote geth node", } - clefURLFlag = cli.StringFlag{ + clefURLFlag = &cli.StringFlag{ Name: "clef", Value: "http://localhost:8550", Usage: "The rpc endpoint of clef", } - signerFlag = cli.StringFlag{ + signerFlag = &cli.StringFlag{ Name: "signer", Usage: "Signer address for clef signing", } - signersFlag = cli.StringFlag{ + signersFlag = &cli.StringFlag{ Name: "signers", Usage: "Comma separated accounts of trusted checkpoint signers", } - signaturesFlag = cli.StringFlag{ + signaturesFlag = &cli.StringFlag{ Name: "signatures", Usage: "Comma separated checkpoint signatures to submit", } diff --git a/cmd/checkpoint-admin/status.go b/cmd/checkpoint-admin/status.go index f613501eb35d6..bec97aed12bd5 100644 --- a/cmd/checkpoint-admin/status.go +++ b/cmd/checkpoint-admin/status.go @@ -19,24 +19,23 @@ package main import ( "fmt" - "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/common" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) -var commandStatus = cli.Command{ +var commandStatus = &cli.Command{ Name: "status", Usage: "Fetches the signers and checkpoint status of the oracle contract", Flags: []cli.Flag{ nodeURLFlag, }, - Action: utils.MigrateFlags(status), + Action: status, } // status fetches the admin list of specified registrar contract. func status(ctx *cli.Context) error { // Create a wrapper around the checkpoint oracle contract - addr, oracle := newContract(newRPCClient(ctx.GlobalString(nodeURLFlag.Name))) + addr, oracle := newContract(newRPCClient(ctx.String(nodeURLFlag.Name))) fmt.Printf("Oracle => %s\n", addr.Hex()) fmt.Println() diff --git a/cmd/clef/main.go b/cmd/clef/main.go index b1ffa38ffefaa..a3e4815ed5faf 100644 --- a/cmd/clef/main.go +++ b/cmd/clef/main.go @@ -30,7 +30,6 @@ import ( "os/signal" "path/filepath" "runtime" - "sort" "strings" "time" @@ -55,7 +54,7 @@ import ( "github.com/ethereum/go-ethereum/signer/storage" "github.com/mattn/go-colorable" "github.com/mattn/go-isatty" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) const legalWarning = ` @@ -73,70 +72,70 @@ PURPOSE. See the GNU General Public License for more details. ` var ( - logLevelFlag = cli.IntFlag{ + logLevelFlag = &cli.IntFlag{ Name: "loglevel", Value: 4, Usage: "log level to emit to the screen", } - advancedMode = cli.BoolFlag{ + advancedMode = &cli.BoolFlag{ Name: "advanced", Usage: "If enabled, issues warnings instead of rejections for suspicious requests. Default off", } - acceptFlag = cli.BoolFlag{ + acceptFlag = &cli.BoolFlag{ Name: "suppress-bootwarn", Usage: "If set, does not show the warning during boot", } - keystoreFlag = cli.StringFlag{ + keystoreFlag = &cli.StringFlag{ Name: "keystore", Value: filepath.Join(node.DefaultDataDir(), "keystore"), Usage: "Directory for the keystore", } - configdirFlag = cli.StringFlag{ + configdirFlag = &cli.StringFlag{ Name: "configdir", Value: DefaultConfigDir(), Usage: "Directory for Clef configuration", } - chainIdFlag = cli.Int64Flag{ + chainIdFlag = &cli.Int64Flag{ Name: "chainid", Value: params.MainnetChainConfig.ChainID.Int64(), Usage: "Chain id to use for signing (1=mainnet, 3=Ropsten, 4=Rinkeby, 5=Goerli)", } - rpcPortFlag = cli.IntFlag{ - Name: "http.port", - Usage: "HTTP-RPC server listening port", - Value: node.DefaultHTTPPort + 5, + rpcPortFlag = &cli.IntFlag{ + Name: "http.port", + Usage: "HTTP-RPC server listening port", + Value: node.DefaultHTTPPort + 5, + Category: flags.APICategory, } - signerSecretFlag = cli.StringFlag{ + signerSecretFlag = &cli.StringFlag{ Name: "signersecret", Usage: "A file containing the (encrypted) master seed to encrypt Clef data, e.g. keystore credentials and ruleset hash", } - customDBFlag = cli.StringFlag{ + customDBFlag = &cli.StringFlag{ Name: "4bytedb-custom", Usage: "File used for writing new 4byte-identifiers submitted via API", Value: "./4byte-custom.json", } - auditLogFlag = cli.StringFlag{ + auditLogFlag = &cli.StringFlag{ Name: "auditlog", Usage: "File used to emit audit logs. Set to \"\" to disable", Value: "audit.log", } - ruleFlag = cli.StringFlag{ + ruleFlag = &cli.StringFlag{ Name: "rules", Usage: "Path to the rule file to auto-authorize requests with", } - stdiouiFlag = cli.BoolFlag{ + stdiouiFlag = &cli.BoolFlag{ Name: "stdio-ui", Usage: "Use STDIN/STDOUT as a channel for an external UI. " + "This means that an STDIN/STDOUT is used for RPC-communication with a e.g. a graphical user " + "interface, and can be used when Clef is started by an external process.", } - testFlag = cli.BoolFlag{ + testFlag = &cli.BoolFlag{ Name: "stdio-ui-test", Usage: "Mechanism to test interface between Clef and UI. Requires 'stdio-ui'.", } - app = cli.NewApp() - initCommand = cli.Command{ - Action: utils.MigrateFlags(initializeSecrets), + initCommand = &cli.Command{ + Action: initializeSecrets, Name: "init", Usage: "Initialize the signer, generate secret storage", ArgsUsage: "", @@ -148,8 +147,8 @@ var ( The init command generates a master seed which Clef can use to store credentials and data needed for the rule-engine to work.`, } - attestCommand = cli.Command{ - Action: utils.MigrateFlags(attestFile), + attestCommand = &cli.Command{ + Action: attestFile, Name: "attest", Usage: "Attest that a js-file is to be used", ArgsUsage: "", @@ -165,8 +164,8 @@ incoming requests. Whenever you make an edit to the rule file, you need to use attestation to tell Clef that the file is 'safe' to execute.`, } - setCredentialCommand = cli.Command{ - Action: utils.MigrateFlags(setCredential), + setCredentialCommand = &cli.Command{ + Action: setCredential, Name: "setpw", Usage: "Store a credential for a keystore file", ArgsUsage: "
", @@ -178,8 +177,8 @@ Clef that the file is 'safe' to execute.`, Description: ` The setpw command stores a password for a given address (keyfile). `} - delCredentialCommand = cli.Command{ - Action: utils.MigrateFlags(removeCredential), + delCredentialCommand = &cli.Command{ + Action: removeCredential, Name: "delpw", Usage: "Remove a credential for a keystore file", ArgsUsage: "
", @@ -191,8 +190,8 @@ The setpw command stores a password for a given address (keyfile). Description: ` The delpw command removes a password for a given address (keyfile). `} - newAccountCommand = cli.Command{ - Action: utils.MigrateFlags(newAccount), + newAccountCommand = &cli.Command{ + Action: newAccount, Name: "newaccount", Usage: "Create a new account", ArgsUsage: "", @@ -207,7 +206,7 @@ The newaccount command creates a new keystore-backed account. It is a convenienc which can be used in lieu of an external UI.`, } - gendocCommand = cli.Command{ + gendocCommand = &cli.Command{ Action: GenDoc, Name: "gendoc", Usage: "Generate documentation about json-rpc format", @@ -216,39 +215,10 @@ The gendoc generates example structures of the json-rpc communication types. `} ) -// AppHelpFlagGroups is the application flags, grouped by functionality. -var AppHelpFlagGroups = []flags.FlagGroup{ - { - Name: "FLAGS", - Flags: []cli.Flag{ - logLevelFlag, - keystoreFlag, - configdirFlag, - chainIdFlag, - utils.LightKDFFlag, - utils.NoUSBFlag, - utils.SmartCardDaemonPathFlag, - utils.HTTPListenAddrFlag, - utils.HTTPVirtualHostsFlag, - utils.IPCDisabledFlag, - utils.IPCPathFlag, - utils.HTTPEnabledFlag, - rpcPortFlag, - signerSecretFlag, - customDBFlag, - auditLogFlag, - ruleFlag, - stdiouiFlag, - testFlag, - advancedMode, - acceptFlag, - }, - }, -} +var app = flags.NewApp("Manage Ethereum account operations") func init() { app.Name = "Clef" - app.Usage = "Manage Ethereum account operations" app.Flags = []cli.Flag{ logLevelFlag, keystoreFlag, @@ -273,46 +243,12 @@ func init() { acceptFlag, } app.Action = signer - app.Commands = []cli.Command{initCommand, + app.Commands = []*cli.Command{initCommand, attestCommand, setCredentialCommand, delCredentialCommand, newAccountCommand, - gendocCommand} - cli.CommandHelpTemplate = flags.CommandHelpTemplate - // Override the default app help template - cli.AppHelpTemplate = flags.ClefAppHelpTemplate - - // Override the default app help printer, but only for the global app help - originalHelpPrinter := cli.HelpPrinter - cli.HelpPrinter = func(w io.Writer, tmpl string, data interface{}) { - if tmpl == flags.ClefAppHelpTemplate { - // Render out custom usage screen - originalHelpPrinter(w, tmpl, flags.HelpData{App: data, FlagGroups: AppHelpFlagGroups}) - } else if tmpl == flags.CommandHelpTemplate { - // Iterate over all command specific flags and categorize them - categorized := make(map[string][]cli.Flag) - for _, flag := range data.(cli.Command).Flags { - if _, ok := categorized[flag.String()]; !ok { - categorized[flags.FlagCategory(flag, AppHelpFlagGroups)] = append(categorized[flags.FlagCategory(flag, AppHelpFlagGroups)], flag) - } - } - - // sort to get a stable ordering - sorted := make([]flags.FlagGroup, 0, len(categorized)) - for cat, flgs := range categorized { - sorted = append(sorted, flags.FlagGroup{Name: cat, Flags: flgs}) - } - sort.Sort(flags.ByCategory(sorted)) - - // add sorted array to data and render with default printer - originalHelpPrinter(w, tmpl, map[string]interface{}{ - "cmd": data, - "categorizedFlags": sorted, - }) - } else { - originalHelpPrinter(w, tmpl, data) - } + gendocCommand, } } @@ -329,7 +265,7 @@ func initializeSecrets(c *cli.Context) error { return err } // Ensure the master key does not yet exist, we're not willing to overwrite - configDir := c.GlobalString(configdirFlag.Name) + configDir := c.String(configdirFlag.Name) if err := os.Mkdir(configDir, 0700); err != nil && !os.IsExist(err) { return err } @@ -347,7 +283,7 @@ func initializeSecrets(c *cli.Context) error { return fmt.Errorf("failed to read enough random") } n, p := keystore.StandardScryptN, keystore.StandardScryptP - if c.GlobalBool(utils.LightKDFFlag.Name) { + if c.Bool(utils.LightKDFFlag.Name) { n, p = keystore.LightScryptN, keystore.LightScryptP } text := "The master seed of clef will be locked with a password.\nPlease specify a password. Do not forget this password!" @@ -390,8 +326,9 @@ You should treat 'masterseed.json' with utmost secrecy and make a backup of it! `) return nil } + func attestFile(ctx *cli.Context) error { - if len(ctx.Args()) < 1 { + if ctx.NArg() < 1 { utils.Fatalf("This command requires an argument.") } if err := initialize(ctx); err != nil { @@ -402,7 +339,7 @@ func attestFile(ctx *cli.Context) error { if err != nil { utils.Fatalf(err.Error()) } - configDir := ctx.GlobalString(configdirFlag.Name) + configDir := ctx.String(configdirFlag.Name) vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), stretchedKey)[:10])) confKey := crypto.Keccak256([]byte("config"), stretchedKey) @@ -415,7 +352,7 @@ func attestFile(ctx *cli.Context) error { } func setCredential(ctx *cli.Context) error { - if len(ctx.Args()) < 1 { + if ctx.NArg() < 1 { utils.Fatalf("This command requires an address to be passed as an argument") } if err := initialize(ctx); err != nil { @@ -433,7 +370,7 @@ func setCredential(ctx *cli.Context) error { if err != nil { utils.Fatalf(err.Error()) } - configDir := ctx.GlobalString(configdirFlag.Name) + configDir := ctx.String(configdirFlag.Name) vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), stretchedKey)[:10])) pwkey := crypto.Keccak256([]byte("credentials"), stretchedKey) @@ -445,7 +382,7 @@ func setCredential(ctx *cli.Context) error { } func removeCredential(ctx *cli.Context) error { - if len(ctx.Args()) < 1 { + if ctx.NArg() < 1 { utils.Fatalf("This command requires an address to be passed as an argument") } if err := initialize(ctx); err != nil { @@ -461,7 +398,7 @@ func removeCredential(ctx *cli.Context) error { if err != nil { utils.Fatalf(err.Error()) } - configDir := ctx.GlobalString(configdirFlag.Name) + configDir := ctx.String(configdirFlag.Name) vaultLocation := filepath.Join(configDir, common.Bytes2Hex(crypto.Keccak256([]byte("vault"), stretchedKey)[:10])) pwkey := crypto.Keccak256([]byte("credentials"), stretchedKey) @@ -481,8 +418,8 @@ func newAccount(c *cli.Context) error { var ( ui = core.NewCommandlineUI() pwStorage storage.Storage = &storage.NoStorage{} - ksLoc = c.GlobalString(keystoreFlag.Name) - lightKdf = c.GlobalBool(utils.LightKDFFlag.Name) + ksLoc = c.String(keystoreFlag.Name) + lightKdf = c.Bool(utils.LightKDFFlag.Name) ) log.Info("Starting clef", "keystore", ksLoc, "light-kdf", lightKdf) am := core.StartClefAccountManager(ksLoc, true, lightKdf, "") @@ -500,13 +437,13 @@ func newAccount(c *cli.Context) error { func initialize(c *cli.Context) error { // Set up the logger to print everything logOutput := os.Stdout - if c.GlobalBool(stdiouiFlag.Name) { + if c.Bool(stdiouiFlag.Name) { logOutput = os.Stderr // If using the stdioui, we can't do the 'confirm'-flow - if !c.GlobalBool(acceptFlag.Name) { + if !c.Bool(acceptFlag.Name) { fmt.Fprint(logOutput, legalWarning) } - } else if !c.GlobalBool(acceptFlag.Name) { + } else if !c.Bool(acceptFlag.Name) { if !confirm(legalWarning) { return fmt.Errorf("aborted by user") } @@ -545,8 +482,8 @@ func ipcEndpoint(ipcPath, datadir string) string { func signer(c *cli.Context) error { // If we have some unrecognized command, bail out - if args := c.Args(); len(args) > 0 { - return fmt.Errorf("invalid command: %q", args[0]) + if c.NArg() > 0 { + return fmt.Errorf("invalid command: %q", c.Args().First()) } if err := initialize(c); err != nil { return err @@ -554,7 +491,7 @@ func signer(c *cli.Context) error { var ( ui core.UIClientAPI ) - if c.GlobalBool(stdiouiFlag.Name) { + if c.Bool(stdiouiFlag.Name) { log.Info("Using stdin/stdout as UI-channel") ui = core.NewStdIOUI() } else { @@ -562,7 +499,7 @@ func signer(c *cli.Context) error { ui = core.NewCommandlineUI() } // 4bytedb data - fourByteLocal := c.GlobalString(customDBFlag.Name) + fourByteLocal := c.String(customDBFlag.Name) db, err := fourbyte.NewWithFile(fourByteLocal) if err != nil { utils.Fatalf(err.Error()) @@ -574,7 +511,7 @@ func signer(c *cli.Context) error { api core.ExternalAPI pwStorage storage.Storage = &storage.NoStorage{} ) - configDir := c.GlobalString(configdirFlag.Name) + configDir := c.String(configdirFlag.Name) if stretchedKey, err := readMasterKey(c, ui); err != nil { log.Warn("Failed to open master, rules disabled", "err", err) } else { @@ -591,7 +528,7 @@ func signer(c *cli.Context) error { configStorage := storage.NewAESEncryptedStorage(filepath.Join(vaultLocation, "config.json"), confkey) // Do we have a rule-file? - if ruleFile := c.GlobalString(ruleFlag.Name); ruleFile != "" { + if ruleFile := c.String(ruleFlag.Name); ruleFile != "" { ruleJS, err := os.ReadFile(ruleFile) if err != nil { log.Warn("Could not load rules, disabling", "file", ruleFile, "err", err) @@ -615,12 +552,12 @@ func signer(c *cli.Context) error { } } var ( - chainId = c.GlobalInt64(chainIdFlag.Name) - ksLoc = c.GlobalString(keystoreFlag.Name) - lightKdf = c.GlobalBool(utils.LightKDFFlag.Name) - advanced = c.GlobalBool(advancedMode.Name) - nousb = c.GlobalBool(utils.NoUSBFlag.Name) - scpath = c.GlobalString(utils.SmartCardDaemonPathFlag.Name) + chainId = c.Int64(chainIdFlag.Name) + ksLoc = c.String(keystoreFlag.Name) + lightKdf = c.Bool(utils.LightKDFFlag.Name) + advanced = c.Bool(advancedMode.Name) + nousb = c.Bool(utils.NoUSBFlag.Name) + scpath = c.String(utils.SmartCardDaemonPathFlag.Name) ) log.Info("Starting signer", "chainid", chainId, "keystore", ksLoc, "light-kdf", lightKdf, "advanced", advanced) @@ -632,7 +569,7 @@ func signer(c *cli.Context) error { ui.RegisterUIServer(core.NewUIServerAPI(apiImpl)) api = apiImpl // Audit logging - if logfile := c.GlobalString(auditLogFlag.Name); logfile != "" { + if logfile := c.String(auditLogFlag.Name); logfile != "" { api, err = core.NewAuditLogger(logfile, api) if err != nil { utils.Fatalf(err.Error()) @@ -647,16 +584,15 @@ func signer(c *cli.Context) error { rpcAPI := []rpc.API{ { Namespace: "account", - Public: true, Service: api, - Version: "1.0"}, + }, } - if c.GlobalBool(utils.HTTPEnabledFlag.Name) { - vhosts := utils.SplitAndTrim(c.GlobalString(utils.HTTPVirtualHostsFlag.Name)) - cors := utils.SplitAndTrim(c.GlobalString(utils.HTTPCORSDomainFlag.Name)) + if c.Bool(utils.HTTPEnabledFlag.Name) { + vhosts := utils.SplitAndTrim(c.String(utils.HTTPVirtualHostsFlag.Name)) + cors := utils.SplitAndTrim(c.String(utils.HTTPCORSDomainFlag.Name)) srv := rpc.NewServer() - err := node.RegisterApis(rpcAPI, []string{"account"}, srv, false) + err := node.RegisterApis(rpcAPI, []string{"account"}, srv) if err != nil { utils.Fatalf("Could not register API: %w", err) } @@ -666,7 +602,7 @@ func signer(c *cli.Context) error { port := c.Int(rpcPortFlag.Name) // start http server - httpEndpoint := fmt.Sprintf("%s:%d", c.GlobalString(utils.HTTPListenAddrFlag.Name), port) + httpEndpoint := fmt.Sprintf("%s:%d", c.String(utils.HTTPListenAddrFlag.Name), port) httpServer, addr, err := node.StartHTTPEndpoint(httpEndpoint, rpc.DefaultHTTPTimeouts, handler) if err != nil { utils.Fatalf("Could not start RPC api: %v", err) @@ -680,8 +616,8 @@ func signer(c *cli.Context) error { log.Info("HTTP endpoint closed", "url", extapiURL) }() } - if !c.GlobalBool(utils.IPCDisabledFlag.Name) { - givenPath := c.GlobalString(utils.IPCPathFlag.Name) + if !c.Bool(utils.IPCDisabledFlag.Name) { + givenPath := c.String(utils.IPCPathFlag.Name) ipcapiURL = ipcEndpoint(filepath.Join(givenPath, "clef.ipc"), configDir) listener, _, err := rpc.StartIPCEndpoint(ipcapiURL, rpcAPI) if err != nil { @@ -694,7 +630,7 @@ func signer(c *cli.Context) error { }() } - if c.GlobalBool(testFlag.Name) { + if c.Bool(testFlag.Name) { log.Info("Performing UI test") go testExternalUI(apiImpl) } @@ -720,7 +656,7 @@ func signer(c *cli.Context) error { // persistence requirements. func DefaultConfigDir() string { // Try to place the data folder in the user's home dir - home := utils.HomeDir() + home := flags.HomeDir() if home != "" { if runtime.GOOS == "darwin" { return filepath.Join(home, "Library", "Signer") @@ -740,10 +676,10 @@ func DefaultConfigDir() string { func readMasterKey(ctx *cli.Context, ui core.UIClientAPI) ([]byte, error) { var ( file string - configDir = ctx.GlobalString(configdirFlag.Name) + configDir = ctx.String(configdirFlag.Name) ) - if ctx.GlobalIsSet(signerSecretFlag.Name) { - file = ctx.GlobalString(signerSecretFlag.Name) + if ctx.IsSet(signerSecretFlag.Name) { + file = ctx.String(signerSecretFlag.Name) } else { file = filepath.Join(configDir, "masterseed.json") } @@ -817,7 +753,6 @@ func confirm(text string) bool { } func testExternalUI(api *core.SignerAPI) { - ctx := context.WithValue(context.Background(), "remote", "clef binary") ctx = context.WithValue(ctx, "scheme", "in-proc") ctx = context.WithValue(ctx, "local", "main") @@ -917,7 +852,6 @@ func testExternalUI(api *core.SignerAPI) { expectDeny("signdata - text", err) } { // Sign transaction - api.UI.ShowInfo("Please reject next transaction") time.Sleep(delay) data := hexutil.Bytes([]byte{}) @@ -960,7 +894,6 @@ func testExternalUI(api *core.SignerAPI) { } result := fmt.Sprintf("Tests completed. %d errors:\n%s\n", len(errs), strings.Join(errs, "\n")) api.UI.ShowInfo(result) - } type encryptedSeedStorage struct { @@ -996,8 +929,7 @@ func decryptSeed(keyjson []byte, auth string) ([]byte, error) { } // GenDoc outputs examples of all structures used in json-rpc communication -func GenDoc(ctx *cli.Context) { - +func GenDoc(ctx *cli.Context) error { var ( a = common.HexToAddress("0xdeadbeef000000000000000000000000deadbeef") b = common.HexToAddress("0x1111111122222222222233333333334444444444") @@ -1107,7 +1039,6 @@ func GenDoc(ctx *cli.Context) { var tx types.Transaction tx.UnmarshalBinary(rlpdata) add("OnApproved - SignTransactionResult", desc, ðapi.SignTransactionResult{Raw: rlpdata, Tx: &tx}) - } { // User input add("UserInputRequest", "Sent when clef needs the user to provide data. If 'password' is true, the input field should be treated accordingly (echo-free)", @@ -1146,4 +1077,5 @@ These data types are defined in the channel between clef and the UI`) for _, elem := range output { fmt.Println(elem) } + return nil } diff --git a/cmd/devp2p/discv4cmd.go b/cmd/devp2p/discv4cmd.go index 3b6dc09a1cc82..9d35880b128b7 100644 --- a/cmd/devp2p/discv4cmd.go +++ b/cmd/devp2p/discv4cmd.go @@ -25,17 +25,18 @@ import ( "github.com/ethereum/go-ethereum/cmd/devp2p/internal/v4test" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/internal/flags" "github.com/ethereum/go-ethereum/p2p/discover" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/params" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) var ( - discv4Command = cli.Command{ + discv4Command = &cli.Command{ Name: "discv4", Usage: "Node Discovery v4 tools", - Subcommands: []cli.Command{ + Subcommands: []*cli.Command{ discv4PingCommand, discv4RequestRecordCommand, discv4ResolveCommand, @@ -44,39 +45,41 @@ var ( discv4TestCommand, }, } - discv4PingCommand = cli.Command{ + discv4PingCommand = &cli.Command{ Name: "ping", Usage: "Sends ping to a node", Action: discv4Ping, ArgsUsage: "", + Flags: v4NodeFlags, } - discv4RequestRecordCommand = cli.Command{ + discv4RequestRecordCommand = &cli.Command{ Name: "requestenr", Usage: "Requests a node record using EIP-868 enrRequest", Action: discv4RequestRecord, ArgsUsage: "", + Flags: v4NodeFlags, } - discv4ResolveCommand = cli.Command{ + discv4ResolveCommand = &cli.Command{ Name: "resolve", Usage: "Finds a node in the DHT", Action: discv4Resolve, ArgsUsage: "", - Flags: []cli.Flag{bootnodesFlag}, + Flags: v4NodeFlags, } - discv4ResolveJSONCommand = cli.Command{ + discv4ResolveJSONCommand = &cli.Command{ Name: "resolve-json", Usage: "Re-resolves nodes in a nodes.json file", Action: discv4ResolveJSON, - Flags: []cli.Flag{bootnodesFlag}, + Flags: v4NodeFlags, ArgsUsage: "", } - discv4CrawlCommand = cli.Command{ + discv4CrawlCommand = &cli.Command{ Name: "crawl", Usage: "Updates a nodes.json file with random nodes found in the DHT", Action: discv4Crawl, - Flags: []cli.Flag{bootnodesFlag, crawlTimeoutFlag}, + Flags: flags.Merge(v4NodeFlags, []cli.Flag{crawlTimeoutFlag}), } - discv4TestCommand = cli.Command{ + discv4TestCommand = &cli.Command{ Name: "test", Usage: "Runs tests against a node", Action: discv4Test, @@ -91,34 +94,41 @@ var ( ) var ( - bootnodesFlag = cli.StringFlag{ + bootnodesFlag = &cli.StringFlag{ Name: "bootnodes", Usage: "Comma separated nodes used for bootstrapping", } - nodekeyFlag = cli.StringFlag{ + nodekeyFlag = &cli.StringFlag{ Name: "nodekey", Usage: "Hex-encoded node key", } - nodedbFlag = cli.StringFlag{ + nodedbFlag = &cli.StringFlag{ Name: "nodedb", Usage: "Nodes database location", } - listenAddrFlag = cli.StringFlag{ + listenAddrFlag = &cli.StringFlag{ Name: "addr", Usage: "Listening address", } - crawlTimeoutFlag = cli.DurationFlag{ + crawlTimeoutFlag = &cli.DurationFlag{ Name: "timeout", Usage: "Time limit for the crawl.", Value: 30 * time.Minute, } - remoteEnodeFlag = cli.StringFlag{ - Name: "remote", - Usage: "Enode of the remote node under test", - EnvVar: "REMOTE_ENODE", + remoteEnodeFlag = &cli.StringFlag{ + Name: "remote", + Usage: "Enode of the remote node under test", + EnvVars: []string{"REMOTE_ENODE"}, } ) +var v4NodeFlags = []cli.Flag{ + bootnodesFlag, + nodekeyFlag, + nodedbFlag, + listenAddrFlag, +} + func discv4Ping(ctx *cli.Context) error { n := getNodeArg(ctx) disc := startV4(ctx) diff --git a/cmd/devp2p/discv5cmd.go b/cmd/devp2p/discv5cmd.go index 873d41e7030cf..298196034b58f 100644 --- a/cmd/devp2p/discv5cmd.go +++ b/cmd/devp2p/discv5cmd.go @@ -23,14 +23,14 @@ import ( "github.com/ethereum/go-ethereum/cmd/devp2p/internal/v5test" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/p2p/discover" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) var ( - discv5Command = cli.Command{ + discv5Command = &cli.Command{ Name: "discv5", Usage: "Node Discovery v5 tools", - Subcommands: []cli.Command{ + Subcommands: []*cli.Command{ discv5PingCommand, discv5ResolveCommand, discv5CrawlCommand, @@ -38,24 +38,24 @@ var ( discv5ListenCommand, }, } - discv5PingCommand = cli.Command{ + discv5PingCommand = &cli.Command{ Name: "ping", Usage: "Sends ping to a node", Action: discv5Ping, } - discv5ResolveCommand = cli.Command{ + discv5ResolveCommand = &cli.Command{ Name: "resolve", Usage: "Finds a node in the DHT", Action: discv5Resolve, Flags: []cli.Flag{bootnodesFlag}, } - discv5CrawlCommand = cli.Command{ + discv5CrawlCommand = &cli.Command{ Name: "crawl", Usage: "Updates a nodes.json file with random nodes found in the DHT", Action: discv5Crawl, Flags: []cli.Flag{bootnodesFlag, crawlTimeoutFlag}, } - discv5TestCommand = cli.Command{ + discv5TestCommand = &cli.Command{ Name: "test", Usage: "Runs protocol tests against a node", Action: discv5Test, @@ -66,7 +66,7 @@ var ( testListen2Flag, }, } - discv5ListenCommand = cli.Command{ + discv5ListenCommand = &cli.Command{ Name: "listen", Usage: "Runs a node", Action: discv5Listen, diff --git a/cmd/devp2p/dns_cloudflare.go b/cmd/devp2p/dns_cloudflare.go index d67aaea1a7fb3..92c6faf272ec4 100644 --- a/cmd/devp2p/dns_cloudflare.go +++ b/cmd/devp2p/dns_cloudflare.go @@ -24,16 +24,16 @@ import ( "github.com/cloudflare/cloudflare-go" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p/dnsdisc" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) var ( - cloudflareTokenFlag = cli.StringFlag{ - Name: "token", - Usage: "CloudFlare API token", - EnvVar: "CLOUDFLARE_API_TOKEN", + cloudflareTokenFlag = &cli.StringFlag{ + Name: "token", + Usage: "CloudFlare API token", + EnvVars: []string{"CLOUDFLARE_API_TOKEN"}, } - cloudflareZoneIDFlag = cli.StringFlag{ + cloudflareZoneIDFlag = &cli.StringFlag{ Name: "zoneid", Usage: "CloudFlare Zone ID (optional)", } @@ -134,7 +134,6 @@ func (c *cloudflareClient) uploadRecords(name string, records map[string]string) ttl := rootTTL if path != name { ttl = treeNodeTTLCloudflare // Max TTL permitted by Cloudflare - } record := cloudflare.DNSRecord{Type: "TXT", Name: path, Content: val, TTL: ttl} _, err = c.CreateDNSRecord(context.Background(), c.zoneID, record) diff --git a/cmd/devp2p/dns_route53.go b/cmd/devp2p/dns_route53.go index 1d4f975dda0b1..4aab0856ff909 100644 --- a/cmd/devp2p/dns_route53.go +++ b/cmd/devp2p/dns_route53.go @@ -32,7 +32,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/route53/types" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p/dnsdisc" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) const ( @@ -45,21 +45,21 @@ const ( ) var ( - route53AccessKeyFlag = cli.StringFlag{ - Name: "access-key-id", - Usage: "AWS Access Key ID", - EnvVar: "AWS_ACCESS_KEY_ID", + route53AccessKeyFlag = &cli.StringFlag{ + Name: "access-key-id", + Usage: "AWS Access Key ID", + EnvVars: []string{"AWS_ACCESS_KEY_ID"}, } - route53AccessSecretFlag = cli.StringFlag{ - Name: "access-key-secret", - Usage: "AWS Access Key Secret", - EnvVar: "AWS_SECRET_ACCESS_KEY", + route53AccessSecretFlag = &cli.StringFlag{ + Name: "access-key-secret", + Usage: "AWS Access Key Secret", + EnvVars: []string{"AWS_SECRET_ACCESS_KEY"}, } - route53ZoneIDFlag = cli.StringFlag{ + route53ZoneIDFlag = &cli.StringFlag{ Name: "zone-id", Usage: "Route53 Zone ID", } - route53RegionFlag = cli.StringFlag{ + route53RegionFlag = &cli.StringFlag{ Name: "aws-region", Usage: "AWS Region", Value: "eu-central-1", diff --git a/cmd/devp2p/dnscmd.go b/cmd/devp2p/dnscmd.go index 21138efdc5ccf..58eb6e8db1c16 100644 --- a/cmd/devp2p/dnscmd.go +++ b/cmd/devp2p/dnscmd.go @@ -29,14 +29,14 @@ import ( "github.com/ethereum/go-ethereum/console/prompt" "github.com/ethereum/go-ethereum/p2p/dnsdisc" "github.com/ethereum/go-ethereum/p2p/enode" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) var ( - dnsCommand = cli.Command{ + dnsCommand = &cli.Command{ Name: "dns", Usage: "DNS Discovery Commands", - Subcommands: []cli.Command{ + Subcommands: []*cli.Command{ dnsSyncCommand, dnsSignCommand, dnsTXTCommand, @@ -45,34 +45,34 @@ var ( dnsRoute53NukeCommand, }, } - dnsSyncCommand = cli.Command{ + dnsSyncCommand = &cli.Command{ Name: "sync", Usage: "Download a DNS discovery tree", ArgsUsage: " [ ]", Action: dnsSync, Flags: []cli.Flag{dnsTimeoutFlag}, } - dnsSignCommand = cli.Command{ + dnsSignCommand = &cli.Command{ Name: "sign", Usage: "Sign a DNS discovery tree", ArgsUsage: " ", Action: dnsSign, Flags: []cli.Flag{dnsDomainFlag, dnsSeqFlag}, } - dnsTXTCommand = cli.Command{ + dnsTXTCommand = &cli.Command{ Name: "to-txt", Usage: "Create a DNS TXT records for a discovery tree", ArgsUsage: " ", Action: dnsToTXT, } - dnsCloudflareCommand = cli.Command{ + dnsCloudflareCommand = &cli.Command{ Name: "to-cloudflare", Usage: "Deploy DNS TXT records to CloudFlare", ArgsUsage: "", Action: dnsToCloudflare, Flags: []cli.Flag{cloudflareTokenFlag, cloudflareZoneIDFlag}, } - dnsRoute53Command = cli.Command{ + dnsRoute53Command = &cli.Command{ Name: "to-route53", Usage: "Deploy DNS TXT records to Amazon Route53", ArgsUsage: "", @@ -84,7 +84,7 @@ var ( route53RegionFlag, }, } - dnsRoute53NukeCommand = cli.Command{ + dnsRoute53NukeCommand = &cli.Command{ Name: "nuke-route53", Usage: "Deletes DNS TXT records of a subdomain on Amazon Route53", ArgsUsage: "", @@ -99,15 +99,15 @@ var ( ) var ( - dnsTimeoutFlag = cli.DurationFlag{ + dnsTimeoutFlag = &cli.DurationFlag{ Name: "timeout", Usage: "Timeout for DNS lookups", } - dnsDomainFlag = cli.StringFlag{ + dnsDomainFlag = &cli.StringFlag{ Name: "domain", Usage: "Domain name of the tree", } - dnsSeqFlag = cli.UintFlag{ + dnsSeqFlag = &cli.UintFlag{ Name: "seq", Usage: "New sequence number of the tree", } diff --git a/cmd/devp2p/enrcmd.go b/cmd/devp2p/enrcmd.go index 2a8f9d508fbe5..2110437103584 100644 --- a/cmd/devp2p/enrcmd.go +++ b/cmd/devp2p/enrcmd.go @@ -30,12 +30,12 @@ import ( "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/p2p/enr" "github.com/ethereum/go-ethereum/rlp" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) -var fileFlag = cli.StringFlag{Name: "file"} +var fileFlag = &cli.StringFlag{Name: "file"} -var enrdumpCommand = cli.Command{ +var enrdumpCommand = &cli.Command{ Name: "enrdump", Usage: "Pretty-prints node records", Action: enrdump, @@ -62,7 +62,7 @@ func enrdump(ctx *cli.Context) error { } source = string(b) } else if ctx.NArg() == 1 { - source = ctx.Args()[0] + source = ctx.Args().First() } else { return fmt.Errorf("need record as argument") } diff --git a/cmd/devp2p/internal/ethtest/chain.go b/cmd/devp2p/internal/ethtest/chain.go index c1d696b40728f..83ceb2a4f2c54 100644 --- a/cmd/devp2p/internal/ethtest/chain.go +++ b/cmd/devp2p/internal/ethtest/chain.go @@ -47,7 +47,7 @@ func (c *Chain) Len() int { // TD calculates the total difficulty of the chain at the // chain head. func (c *Chain) TD() *big.Int { - sum := big.NewInt(0) + sum := new(big.Int) for _, block := range c.blocks[:c.Len()] { sum.Add(sum, block.Difficulty()) } @@ -57,7 +57,7 @@ func (c *Chain) TD() *big.Int { // TotalDifficultyAt calculates the total difficulty of the chain // at the given block height. func (c *Chain) TotalDifficultyAt(height int) *big.Int { - sum := big.NewInt(0) + sum := new(big.Int) if height >= c.Len() { return sum } @@ -96,12 +96,12 @@ func (c *Chain) Head() *types.Block { return c.blocks[c.Len()-1] } -func (c *Chain) GetHeaders(req GetBlockHeaders) (BlockHeaders, error) { +func (c *Chain) GetHeaders(req *GetBlockHeaders) ([]*types.Header, error) { if req.Amount < 1 { return nil, fmt.Errorf("no block headers requested") } - headers := make(BlockHeaders, req.Amount) + headers := make([]*types.Header, req.Amount) var blockNumber uint64 // range over blocks to check if our chain has the requested header @@ -119,7 +119,6 @@ func (c *Chain) GetHeaders(req GetBlockHeaders) (BlockHeaders, error) { for i := 1; i < int(req.Amount); i++ { blockNumber -= (1 - req.Skip) headers[i] = c.blocks[blockNumber].Header() - } return headers, nil @@ -140,7 +139,7 @@ func loadChain(chainfile string, genesis string) (*Chain, error) { if err != nil { return nil, err } - gblock := gen.ToBlock(nil) + gblock := gen.ToBlock() blocks, err := blocksFromFile(chainfile, gblock) if err != nil { diff --git a/cmd/devp2p/internal/ethtest/chain_test.go b/cmd/devp2p/internal/ethtest/chain_test.go index 0f232b150611a..67221923a6844 100644 --- a/cmd/devp2p/internal/ethtest/chain_test.go +++ b/cmd/devp2p/internal/ethtest/chain_test.go @@ -21,6 +21,7 @@ import ( "strconv" "testing" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth/protocols/eth" "github.com/ethereum/go-ethereum/p2p" "github.com/stretchr/testify/assert" @@ -140,18 +141,18 @@ func TestChain_GetHeaders(t *testing.T) { var tests = []struct { req GetBlockHeaders - expected BlockHeaders + expected []*types.Header }{ { req: GetBlockHeaders{ - Origin: eth.HashOrNumber{ - Number: uint64(2), + GetBlockHeadersPacket: ð.GetBlockHeadersPacket{ + Origin: eth.HashOrNumber{Number: uint64(2)}, + Amount: uint64(5), + Skip: 1, + Reverse: false, }, - Amount: uint64(5), - Skip: 1, - Reverse: false, }, - expected: BlockHeaders{ + expected: []*types.Header{ chain.blocks[2].Header(), chain.blocks[4].Header(), chain.blocks[6].Header(), @@ -161,14 +162,14 @@ func TestChain_GetHeaders(t *testing.T) { }, { req: GetBlockHeaders{ - Origin: eth.HashOrNumber{ - Number: uint64(chain.Len() - 1), + GetBlockHeadersPacket: ð.GetBlockHeadersPacket{ + Origin: eth.HashOrNumber{Number: uint64(chain.Len() - 1)}, + Amount: uint64(3), + Skip: 0, + Reverse: true, }, - Amount: uint64(3), - Skip: 0, - Reverse: true, }, - expected: BlockHeaders{ + expected: []*types.Header{ chain.blocks[chain.Len()-1].Header(), chain.blocks[chain.Len()-2].Header(), chain.blocks[chain.Len()-3].Header(), @@ -176,14 +177,14 @@ func TestChain_GetHeaders(t *testing.T) { }, { req: GetBlockHeaders{ - Origin: eth.HashOrNumber{ - Hash: chain.Head().Hash(), + GetBlockHeadersPacket: ð.GetBlockHeadersPacket{ + Origin: eth.HashOrNumber{Hash: chain.Head().Hash()}, + Amount: uint64(1), + Skip: 0, + Reverse: false, }, - Amount: uint64(1), - Skip: 0, - Reverse: false, }, - expected: BlockHeaders{ + expected: []*types.Header{ chain.Head().Header(), }, }, @@ -191,7 +192,7 @@ func TestChain_GetHeaders(t *testing.T) { for i, tt := range tests { t.Run(strconv.Itoa(i), func(t *testing.T) { - headers, err := chain.GetHeaders(tt.req) + headers, err := chain.GetHeaders(&tt.req) if err != nil { t.Fatal(err) } diff --git a/cmd/devp2p/internal/ethtest/helpers.go b/cmd/devp2p/internal/ethtest/helpers.go index df754d6ce61dc..b57649ade99d5 100644 --- a/cmd/devp2p/internal/ethtest/helpers.go +++ b/cmd/devp2p/internal/ethtest/helpers.go @@ -43,21 +43,6 @@ var ( timeout = 20 * time.Second ) -// Is_66 checks if the node supports the eth66 protocol version, -// and if not, exists the test suite -func (s *Suite) Is_66(t *utesting.T) { - conn, err := s.dial66() - if err != nil { - t.Fatalf("dial failed: %v", err) - } - if err := conn.handshake(); err != nil { - t.Fatalf("handshake failed: %v", err) - } - if conn.negotiatedProtoVersion < 66 { - t.Fail() - } -} - // dial attempts to dial the given node and perform a handshake, // returning the created Conn if successful. func (s *Suite) dial() (*Conn, error) { @@ -76,31 +61,16 @@ func (s *Suite) dial() (*Conn, error) { } // set default p2p capabilities conn.caps = []p2p.Cap{ - {Name: "eth", Version: 64}, - {Name: "eth", Version: 65}, + {Name: "eth", Version: 66}, + {Name: "eth", Version: 67}, } - conn.ourHighestProtoVersion = 65 + conn.ourHighestProtoVersion = 67 return &conn, nil } -// dial66 attempts to dial the given node and perform a handshake, -// returning the created Conn with additional eth66 capabilities if -// successful -func (s *Suite) dial66() (*Conn, error) { - conn, err := s.dial() - if err != nil { - return nil, fmt.Errorf("dial failed: %v", err) - } - conn.caps = append(conn.caps, p2p.Cap{Name: "eth", Version: 66}) - conn.ourHighestProtoVersion = 66 - return conn, nil -} - -// dial66 attempts to dial the given node and perform a handshake, -// returning the created Conn with additional snap/1 capabilities if -// successful. +// dialSnap creates a connection with snap/1 capability. func (s *Suite) dialSnap() (*Conn, error) { - conn, err := s.dial66() + conn, err := s.dial() if err != nil { return nil, fmt.Errorf("dial failed: %v", err) } @@ -235,117 +205,68 @@ loop: // createSendAndRecvConns creates two connections, one for sending messages to the // node, and one for receiving messages from the node. -func (s *Suite) createSendAndRecvConns(isEth66 bool) (*Conn, *Conn, error) { - var ( - sendConn *Conn - recvConn *Conn - err error - ) - if isEth66 { - sendConn, err = s.dial66() - if err != nil { - return nil, nil, fmt.Errorf("dial failed: %v", err) - } - recvConn, err = s.dial66() - if err != nil { - sendConn.Close() - return nil, nil, fmt.Errorf("dial failed: %v", err) - } - } else { - sendConn, err = s.dial() - if err != nil { - return nil, nil, fmt.Errorf("dial failed: %v", err) - } - recvConn, err = s.dial() - if err != nil { - sendConn.Close() - return nil, nil, fmt.Errorf("dial failed: %v", err) - } +func (s *Suite) createSendAndRecvConns() (*Conn, *Conn, error) { + sendConn, err := s.dial() + if err != nil { + return nil, nil, fmt.Errorf("dial failed: %v", err) } - return sendConn, recvConn, nil -} - -func (c *Conn) readAndServe(chain *Chain, timeout time.Duration) Message { - if c.negotiatedProtoVersion == 66 { - _, msg := c.readAndServe66(chain, timeout) - return msg + recvConn, err := s.dial() + if err != nil { + sendConn.Close() + return nil, nil, fmt.Errorf("dial failed: %v", err) } - return c.readAndServe65(chain, timeout) + return sendConn, recvConn, nil } // readAndServe serves GetBlockHeaders requests while waiting // on another message from the node. -func (c *Conn) readAndServe65(chain *Chain, timeout time.Duration) Message { - start := time.Now() - for time.Since(start) < timeout { - c.SetReadDeadline(time.Now().Add(5 * time.Second)) - switch msg := c.Read().(type) { - case *Ping: - c.Write(&Pong{}) - case *GetBlockHeaders: - req := *msg - headers, err := chain.GetHeaders(req) - if err != nil { - return errorf("could not get headers for inbound header request: %v", err) - } - if err := c.Write(headers); err != nil { - return errorf("could not write to connection: %v", err) - } - default: - return msg - } - } - return errorf("no message received within %v", timeout) -} - -// readAndServe66 serves eth66 GetBlockHeaders requests while waiting -// on another message from the node. -func (c *Conn) readAndServe66(chain *Chain, timeout time.Duration) (uint64, Message) { +func (c *Conn) readAndServe(chain *Chain, timeout time.Duration) Message { start := time.Now() for time.Since(start) < timeout { c.SetReadDeadline(time.Now().Add(10 * time.Second)) - reqID, msg := c.Read66() - + msg := c.Read() switch msg := msg.(type) { case *Ping: c.Write(&Pong{}) - case GetBlockHeaders: + case *GetBlockHeaders: headers, err := chain.GetHeaders(msg) if err != nil { - return 0, errorf("could not get headers for inbound header request: %v", err) + return errorf("could not get headers for inbound header request: %v", err) } - resp := ð.BlockHeadersPacket66{ - RequestId: reqID, + resp := &BlockHeaders{ + RequestId: msg.ReqID(), BlockHeadersPacket: eth.BlockHeadersPacket(headers), } - if err := c.Write66(resp, BlockHeaders{}.Code()); err != nil { - return 0, errorf("could not write to connection: %v", err) + if err := c.Write(resp); err != nil { + return errorf("could not write to connection: %v", err) } default: - return reqID, msg + return msg } } - return 0, errorf("no message received within %v", timeout) + return errorf("no message received within %v", timeout) } // headersRequest executes the given `GetBlockHeaders` request. -func (c *Conn) headersRequest(request *GetBlockHeaders, chain *Chain, isEth66 bool, reqID uint64) (BlockHeaders, error) { +func (c *Conn) headersRequest(request *GetBlockHeaders, chain *Chain, reqID uint64) ([]*types.Header, error) { defer c.SetReadDeadline(time.Time{}) c.SetReadDeadline(time.Now().Add(20 * time.Second)) - // if on eth66 connection, perform eth66 GetBlockHeaders request - if isEth66 { - return getBlockHeaders66(chain, c, request, reqID) - } + + // write request + request.RequestId = reqID if err := c.Write(request); err != nil { - return nil, err + return nil, fmt.Errorf("could not write to connection: %v", err) } - switch msg := c.readAndServe(chain, timeout).(type) { - case *BlockHeaders: - return *msg, nil - default: - return nil, fmt.Errorf("invalid message: %s", pretty.Sdump(msg)) + + // wait for response + msg := c.waitForResponse(chain, timeout, request.RequestId) + resp, ok := msg.(*BlockHeaders) + if !ok { + return nil, fmt.Errorf("unexpected message received: %s", pretty.Sdump(msg)) } + headers := []*types.Header(resp.BlockHeadersPacket) + return headers, nil } func (c *Conn) snapRequest(msg Message, id uint64, chain *Chain) (Message, error) { @@ -357,28 +278,8 @@ func (c *Conn) snapRequest(msg Message, id uint64, chain *Chain) (Message, error return c.ReadSnap(id) } -// getBlockHeaders66 executes the given `GetBlockHeaders` request over the eth66 protocol. -func getBlockHeaders66(chain *Chain, conn *Conn, request *GetBlockHeaders, id uint64) (BlockHeaders, error) { - // write request - packet := eth.GetBlockHeadersPacket(*request) - req := ð.GetBlockHeadersPacket66{ - RequestId: id, - GetBlockHeadersPacket: &packet, - } - if err := conn.Write66(req, GetBlockHeaders{}.Code()); err != nil { - return nil, fmt.Errorf("could not write to connection: %v", err) - } - // wait for response - msg := conn.waitForResponse(chain, timeout, req.RequestId) - headers, ok := msg.(BlockHeaders) - if !ok { - return nil, fmt.Errorf("unexpected message received: %s", pretty.Sdump(msg)) - } - return headers, nil -} - // headersMatch returns whether the received headers match the given request -func headersMatch(expected BlockHeaders, headers BlockHeaders) bool { +func headersMatch(expected []*types.Header, headers []*types.Header) bool { return reflect.DeepEqual(expected, headers) } @@ -386,8 +287,8 @@ func headersMatch(expected BlockHeaders, headers BlockHeaders) bool { // request ID is received. func (c *Conn) waitForResponse(chain *Chain, timeout time.Duration, requestID uint64) Message { for { - id, msg := c.readAndServe66(chain, timeout) - if id == requestID { + msg := c.readAndServe(chain, timeout) + if msg.ReqID() == requestID { return msg } } @@ -395,9 +296,9 @@ func (c *Conn) waitForResponse(chain *Chain, timeout time.Duration, requestID ui // sendNextBlock broadcasts the next block in the chain and waits // for the node to propagate the block and import it into its chain. -func (s *Suite) sendNextBlock(isEth66 bool) error { +func (s *Suite) sendNextBlock() error { // set up sending and receiving connections - sendConn, recvConn, err := s.createSendAndRecvConns(isEth66) + sendConn, recvConn, err := s.createSendAndRecvConns() if err != nil { return err } @@ -420,7 +321,7 @@ func (s *Suite) sendNextBlock(isEth66 bool) error { return fmt.Errorf("failed to announce block: %v", err) } // wait for client to update its chain - if err = s.waitForBlockImport(recvConn, nextBlock, isEth66); err != nil { + if err = s.waitForBlockImport(recvConn, nextBlock); err != nil { return fmt.Errorf("failed to receive confirmation of block import: %v", err) } // update test suite chain @@ -456,38 +357,35 @@ func (s *Suite) waitAnnounce(conn *Conn, blockAnnouncement *NewBlock) error { return fmt.Errorf("wrong block hash in announcement: expected %v, got %v", blockAnnouncement.Block.Hash(), hashes[0].Hash) } return nil + + // ignore tx announcements from previous tests case *NewPooledTransactionHashes: - // ignore tx announcements from previous tests continue + case *Transactions: + continue + default: return fmt.Errorf("unexpected: %s", pretty.Sdump(msg)) } } } -func (s *Suite) waitForBlockImport(conn *Conn, block *types.Block, isEth66 bool) error { +func (s *Suite) waitForBlockImport(conn *Conn, block *types.Block) error { defer conn.SetReadDeadline(time.Time{}) conn.SetReadDeadline(time.Now().Add(20 * time.Second)) // create request req := &GetBlockHeaders{ - Origin: eth.HashOrNumber{ - Hash: block.Hash(), + GetBlockHeadersPacket: ð.GetBlockHeadersPacket{ + Origin: eth.HashOrNumber{Hash: block.Hash()}, + Amount: 1, }, - Amount: 1, } + // loop until BlockHeaders response contains desired block, confirming the // node imported the block for { - var ( - headers BlockHeaders - err error - ) - if isEth66 { - requestID := uint64(54) - headers, err = conn.headersRequest(req, s.chain, eth66, requestID) - } else { - headers, err = conn.headersRequest(req, s.chain, eth65, 0) - } + requestID := uint64(54) + headers, err := conn.headersRequest(req, s.chain, requestID) if err != nil { return fmt.Errorf("GetBlockHeader request failed: %v", err) } @@ -503,8 +401,8 @@ func (s *Suite) waitForBlockImport(conn *Conn, block *types.Block, isEth66 bool) } } -func (s *Suite) oldAnnounce(isEth66 bool) error { - sendConn, receiveConn, err := s.createSendAndRecvConns(isEth66) +func (s *Suite) oldAnnounce() error { + sendConn, receiveConn, err := s.createSendAndRecvConns() if err != nil { return err } @@ -550,23 +448,13 @@ func (s *Suite) oldAnnounce(isEth66 bool) error { return nil } -func (s *Suite) maliciousHandshakes(t *utesting.T, isEth66 bool) error { - var ( - conn *Conn - err error - ) - if isEth66 { - conn, err = s.dial66() - if err != nil { - return fmt.Errorf("dial failed: %v", err) - } - } else { - conn, err = s.dial() - if err != nil { - return fmt.Errorf("dial failed: %v", err) - } +func (s *Suite) maliciousHandshakes(t *utesting.T) error { + conn, err := s.dial() + if err != nil { + return fmt.Errorf("dial failed: %v", err) } defer conn.Close() + // write hello to client pub0 := crypto.FromECDSAPub(&conn.ourKey.PublicKey)[1:] handshakes := []*Hello{ @@ -627,16 +515,9 @@ func (s *Suite) maliciousHandshakes(t *utesting.T, isEth66 bool) error { } } // dial for the next round - if isEth66 { - conn, err = s.dial66() - if err != nil { - return fmt.Errorf("dial failed: %v", err) - } - } else { - conn, err = s.dial() - if err != nil { - return fmt.Errorf("dial failed: %v", err) - } + conn, err = s.dial() + if err != nil { + return fmt.Errorf("dial failed: %v", err) } } return nil @@ -654,6 +535,7 @@ func (s *Suite) maliciousStatus(conn *Conn) error { Genesis: s.chain.blocks[0].Hash(), ForkID: s.chain.ForkID(), } + // get status msg, err := conn.statusExchange(s.chain, status) if err != nil { @@ -664,6 +546,7 @@ func (s *Suite) maliciousStatus(conn *Conn) error { default: return fmt.Errorf("expected status, got: %#v ", msg) } + // wait for disconnect switch msg := conn.readAndServe(s.chain, timeout).(type) { case *Disconnect: @@ -675,9 +558,9 @@ func (s *Suite) maliciousStatus(conn *Conn) error { } } -func (s *Suite) hashAnnounce(isEth66 bool) error { +func (s *Suite) hashAnnounce() error { // create connections - sendConn, recvConn, err := s.createSendAndRecvConns(isEth66) + sendConn, recvConn, err := s.createSendAndRecvConns() if err != nil { return fmt.Errorf("failed to create connections: %v", err) } @@ -689,6 +572,7 @@ func (s *Suite) hashAnnounce(isEth66 bool) error { if err := recvConn.peer(s.chain, nil); err != nil { return fmt.Errorf("peering failed: %v", err) } + // create NewBlockHashes announcement type anno struct { Hash common.Hash // Hash of one particular block being announced @@ -700,56 +584,29 @@ func (s *Suite) hashAnnounce(isEth66 bool) error { if err := sendConn.Write(newBlockHash); err != nil { return fmt.Errorf("failed to write to connection: %v", err) } + // Announcement sent, now wait for a header request - var ( - id uint64 - msg Message - blockHeaderReq GetBlockHeaders - ) - if isEth66 { - id, msg = sendConn.Read66() - switch msg := msg.(type) { - case GetBlockHeaders: - blockHeaderReq = msg - default: - return fmt.Errorf("unexpected %s", pretty.Sdump(msg)) - } - if blockHeaderReq.Amount != 1 { - return fmt.Errorf("unexpected number of block headers requested: %v", blockHeaderReq.Amount) - } - if blockHeaderReq.Origin.Hash != announcement.Hash { - return fmt.Errorf("unexpected block header requested. Announced:\n %v\n Remote request:\n%v", - pretty.Sdump(announcement), - pretty.Sdump(blockHeaderReq)) - } - if err := sendConn.Write66(ð.BlockHeadersPacket66{ - RequestId: id, - BlockHeadersPacket: eth.BlockHeadersPacket{ - nextBlock.Header(), - }, - }, BlockHeaders{}.Code()); err != nil { - return fmt.Errorf("failed to write to connection: %v", err) - } - } else { - msg = sendConn.Read() - switch msg := msg.(type) { - case *GetBlockHeaders: - blockHeaderReq = *msg - default: - return fmt.Errorf("unexpected %s", pretty.Sdump(msg)) - } - if blockHeaderReq.Amount != 1 { - return fmt.Errorf("unexpected number of block headers requested: %v", blockHeaderReq.Amount) - } - if blockHeaderReq.Origin.Hash != announcement.Hash { - return fmt.Errorf("unexpected block header requested. Announced:\n %v\n Remote request:\n%v", - pretty.Sdump(announcement), - pretty.Sdump(blockHeaderReq)) - } - if err := sendConn.Write(&BlockHeaders{nextBlock.Header()}); err != nil { - return fmt.Errorf("failed to write to connection: %v", err) - } + msg := sendConn.Read() + blockHeaderReq, ok := msg.(*GetBlockHeaders) + if !ok { + return fmt.Errorf("unexpected %s", pretty.Sdump(msg)) } + if blockHeaderReq.Amount != 1 { + return fmt.Errorf("unexpected number of block headers requested: %v", blockHeaderReq.Amount) + } + if blockHeaderReq.Origin.Hash != announcement.Hash { + return fmt.Errorf("unexpected block header requested. Announced:\n %v\n Remote request:\n%v", + pretty.Sdump(announcement), + pretty.Sdump(blockHeaderReq)) + } + err = sendConn.Write(&BlockHeaders{ + RequestId: blockHeaderReq.ReqID(), + BlockHeadersPacket: eth.BlockHeadersPacket{nextBlock.Header()}, + }) + if err != nil { + return fmt.Errorf("failed to write to connection: %v", err) + } + // wait for block announcement msg = recvConn.readAndServe(s.chain, timeout) switch msg := msg.(type) { @@ -762,6 +619,7 @@ func (s *Suite) hashAnnounce(isEth66 bool) error { return fmt.Errorf("unexpected block hash announcement, wanted %v, got %v", nextBlock.Hash(), hashes[0].Hash) } + case *NewBlock: // node should only propagate NewBlock without having requested the body if the body is empty nextBlockBody := nextBlock.Body() @@ -780,7 +638,7 @@ func (s *Suite) hashAnnounce(isEth66 bool) error { return fmt.Errorf("unexpected: %s", pretty.Sdump(msg)) } // confirm node imported block - if err := s.waitForBlockImport(recvConn, nextBlock, isEth66); err != nil { + if err := s.waitForBlockImport(recvConn, nextBlock); err != nil { return fmt.Errorf("error waiting for node to import new block: %v", err) } // update the chain diff --git a/cmd/devp2p/internal/ethtest/snap.go b/cmd/devp2p/internal/ethtest/snap.go index 2bfd29c75abf1..032afeafcdad6 100644 --- a/cmd/devp2p/internal/ethtest/snap.go +++ b/cmd/devp2p/internal/ethtest/snap.go @@ -104,6 +104,7 @@ func (s *Suite) TestSnapGetAccountRange(t *utesting.T) { // Max bytes: 0. Expect to deliver one account. {0, root, zero, ffHash, 1, firstKey, firstKey}, } { + tc := tc if err := s.snapGetAccountRange(t, &tc); err != nil { t.Errorf("test %d \n root: %x\n range: %#x - %#x\n bytes: %d\nfailed: %v", i, tc.root, tc.origin, tc.limit, tc.nBytes, err) } @@ -194,6 +195,7 @@ func (s *Suite) TestSnapGetStorageRanges(t *utesting.T) { expSlots: 2, }, } { + tc := tc if err := s.snapGetStorageRanges(t, &tc); err != nil { t.Errorf("test %d \n root: %x\n range: %#x - %#x\n bytes: %d\n #accounts: %d\nfailed: %v", i, tc.root, tc.origin, tc.limit, tc.nBytes, len(tc.accounts), err) @@ -291,6 +293,7 @@ func (s *Suite) TestSnapGetByteCodes(t *utesting.T) { expHashes: 4, }, } { + tc := tc if err := s.snapGetByteCodes(t, &tc); err != nil { t.Errorf("test %d \n bytes: %d\n #hashes: %d\nfailed: %v", i, tc.nBytes, len(tc.hashes), err) } @@ -347,7 +350,6 @@ func hexToCompact(hex []byte) []byte { // TestSnapTrieNodes various forms of GetTrieNodes requests. func (s *Suite) TestSnapTrieNodes(t *utesting.T) { - key := common.FromHex("0x00bf49f440a1cd0527e4d06e2765654c0f56452257516d793a9b8d604dcfdf2a") // helper function to iterate the key, and generate the compact-encoded // trie paths along the way. @@ -436,6 +438,7 @@ func (s *Suite) TestSnapTrieNodes(t *utesting.T) { }, }, } { + tc := tc if err := s.snapGetTrieNodes(t, &tc); err != nil { t.Errorf("test %d \n #hashes %x\n root: %#x\n bytes: %d\nfailed: %v", i, len(tc.expHashes), tc.root, tc.nBytes, err) } @@ -492,10 +495,10 @@ func (s *Suite) snapGetAccountRange(t *utesting.T, tc *accRangeTest) error { } if len(hashes) > 0 { if exp, got := tc.expFirst, res.Accounts[0].Hash; exp != got { - return fmt.Errorf("expected first account 0x%x, got 0x%x", exp, got) + return fmt.Errorf("expected first account %#x, got %#x", exp, got) } if exp, got := tc.expLast, res.Accounts[len(res.Accounts)-1].Hash; exp != got { - return fmt.Errorf("expected last account 0x%x, got 0x%x", exp, got) + return fmt.Errorf("expected last account %#x, got %#x", exp, got) } } // Reconstruct a partial trie from the response and verify it diff --git a/cmd/devp2p/internal/ethtest/snapTypes.go b/cmd/devp2p/internal/ethtest/snapTypes.go index e18cd5925cbb6..6bcaa9291ab22 100644 --- a/cmd/devp2p/internal/ethtest/snapTypes.go +++ b/cmd/devp2p/internal/ethtest/snapTypes.go @@ -21,32 +21,40 @@ import "github.com/ethereum/go-ethereum/eth/protocols/snap" // GetAccountRange represents an account range query. type GetAccountRange snap.GetAccountRangePacket -func (g GetAccountRange) Code() int { return 33 } +func (msg GetAccountRange) Code() int { return 33 } +func (msg GetAccountRange) ReqID() uint64 { return msg.ID } type AccountRange snap.AccountRangePacket -func (g AccountRange) Code() int { return 34 } +func (msg AccountRange) Code() int { return 34 } +func (msg AccountRange) ReqID() uint64 { return msg.ID } type GetStorageRanges snap.GetStorageRangesPacket -func (g GetStorageRanges) Code() int { return 35 } +func (msg GetStorageRanges) Code() int { return 35 } +func (msg GetStorageRanges) ReqID() uint64 { return msg.ID } type StorageRanges snap.StorageRangesPacket -func (g StorageRanges) Code() int { return 36 } +func (msg StorageRanges) Code() int { return 36 } +func (msg StorageRanges) ReqID() uint64 { return msg.ID } type GetByteCodes snap.GetByteCodesPacket -func (g GetByteCodes) Code() int { return 37 } +func (msg GetByteCodes) Code() int { return 37 } +func (msg GetByteCodes) ReqID() uint64 { return msg.ID } type ByteCodes snap.ByteCodesPacket -func (g ByteCodes) Code() int { return 38 } +func (msg ByteCodes) Code() int { return 38 } +func (msg ByteCodes) ReqID() uint64 { return msg.ID } type GetTrieNodes snap.GetTrieNodesPacket -func (g GetTrieNodes) Code() int { return 39 } +func (msg GetTrieNodes) Code() int { return 39 } +func (msg GetTrieNodes) ReqID() uint64 { return msg.ID } type TrieNodes snap.TrieNodesPacket -func (g TrieNodes) Code() int { return 40 } +func (msg TrieNodes) Code() int { return 40 } +func (msg TrieNodes) ReqID() uint64 { return msg.ID } diff --git a/cmd/devp2p/internal/ethtest/suite.go b/cmd/devp2p/internal/ethtest/suite.go index 4ddd65b95865b..4497478d72d65 100644 --- a/cmd/devp2p/internal/ethtest/suite.go +++ b/cmd/devp2p/internal/ethtest/suite.go @@ -49,79 +49,30 @@ func NewSuite(dest *enode.Node, chainfile string, genesisfile string) (*Suite, e }, nil } -func (s *Suite) AllEthTests() []utesting.Test { +func (s *Suite) EthTests() []utesting.Test { return []utesting.Test{ // status - {Name: "TestStatus65", Fn: s.TestStatus65}, - {Name: "TestStatus66", Fn: s.TestStatus66}, + {Name: "TestStatus", Fn: s.TestStatus}, // get block headers - {Name: "TestGetBlockHeaders65", Fn: s.TestGetBlockHeaders65}, - {Name: "TestGetBlockHeaders66", Fn: s.TestGetBlockHeaders66}, - {Name: "TestSimultaneousRequests66", Fn: s.TestSimultaneousRequests66}, - {Name: "TestSameRequestID66", Fn: s.TestSameRequestID66}, - {Name: "TestZeroRequestID66", Fn: s.TestZeroRequestID66}, + {Name: "TestGetBlockHeaders", Fn: s.TestGetBlockHeaders}, + {Name: "TestSimultaneousRequests", Fn: s.TestSimultaneousRequests}, + {Name: "TestSameRequestID", Fn: s.TestSameRequestID}, + {Name: "TestZeroRequestID", Fn: s.TestZeroRequestID}, // get block bodies - {Name: "TestGetBlockBodies65", Fn: s.TestGetBlockBodies65}, - {Name: "TestGetBlockBodies66", Fn: s.TestGetBlockBodies66}, + {Name: "TestGetBlockBodies", Fn: s.TestGetBlockBodies}, // broadcast - {Name: "TestBroadcast65", Fn: s.TestBroadcast65}, - {Name: "TestBroadcast66", Fn: s.TestBroadcast66}, - {Name: "TestLargeAnnounce65", Fn: s.TestLargeAnnounce65}, - {Name: "TestLargeAnnounce66", Fn: s.TestLargeAnnounce66}, - {Name: "TestOldAnnounce65", Fn: s.TestOldAnnounce65}, - {Name: "TestOldAnnounce66", Fn: s.TestOldAnnounce66}, - {Name: "TestBlockHashAnnounce65", Fn: s.TestBlockHashAnnounce65}, - {Name: "TestBlockHashAnnounce66", Fn: s.TestBlockHashAnnounce66}, + {Name: "TestBroadcast", Fn: s.TestBroadcast}, + {Name: "TestLargeAnnounce", Fn: s.TestLargeAnnounce}, + {Name: "TestOldAnnounce", Fn: s.TestOldAnnounce}, + {Name: "TestBlockHashAnnounce", Fn: s.TestBlockHashAnnounce}, // malicious handshakes + status - {Name: "TestMaliciousHandshake65", Fn: s.TestMaliciousHandshake65}, - {Name: "TestMaliciousStatus65", Fn: s.TestMaliciousStatus65}, - {Name: "TestMaliciousHandshake66", Fn: s.TestMaliciousHandshake66}, - {Name: "TestMaliciousStatus66", Fn: s.TestMaliciousStatus66}, + {Name: "TestMaliciousHandshake", Fn: s.TestMaliciousHandshake}, + {Name: "TestMaliciousStatus", Fn: s.TestMaliciousStatus}, // test transactions - {Name: "TestTransaction65", Fn: s.TestTransaction65}, - {Name: "TestTransaction66", Fn: s.TestTransaction66}, - {Name: "TestMaliciousTx65", Fn: s.TestMaliciousTx65}, - {Name: "TestMaliciousTx66", Fn: s.TestMaliciousTx66}, - {Name: "TestLargeTxRequest66", Fn: s.TestLargeTxRequest66}, - {Name: "TestNewPooledTxs66", Fn: s.TestNewPooledTxs66}, - } -} - -func (s *Suite) EthTests() []utesting.Test { - return []utesting.Test{ - {Name: "TestStatus65", Fn: s.TestStatus65}, - {Name: "TestGetBlockHeaders65", Fn: s.TestGetBlockHeaders65}, - {Name: "TestGetBlockBodies65", Fn: s.TestGetBlockBodies65}, - {Name: "TestBroadcast65", Fn: s.TestBroadcast65}, - {Name: "TestLargeAnnounce65", Fn: s.TestLargeAnnounce65}, - {Name: "TestOldAnnounce65", Fn: s.TestOldAnnounce65}, - {Name: "TestBlockHashAnnounce65", Fn: s.TestBlockHashAnnounce65}, - {Name: "TestMaliciousHandshake65", Fn: s.TestMaliciousHandshake65}, - {Name: "TestMaliciousStatus65", Fn: s.TestMaliciousStatus65}, - {Name: "TestTransaction65", Fn: s.TestTransaction65}, - {Name: "TestMaliciousTx65", Fn: s.TestMaliciousTx65}, - } -} - -func (s *Suite) Eth66Tests() []utesting.Test { - return []utesting.Test{ - // only proceed with eth66 test suite if node supports eth 66 protocol - {Name: "TestStatus66", Fn: s.TestStatus66}, - {Name: "TestGetBlockHeaders66", Fn: s.TestGetBlockHeaders66}, - {Name: "TestSimultaneousRequests66", Fn: s.TestSimultaneousRequests66}, - {Name: "TestSameRequestID66", Fn: s.TestSameRequestID66}, - {Name: "TestZeroRequestID66", Fn: s.TestZeroRequestID66}, - {Name: "TestGetBlockBodies66", Fn: s.TestGetBlockBodies66}, - {Name: "TestBroadcast66", Fn: s.TestBroadcast66}, - {Name: "TestLargeAnnounce66", Fn: s.TestLargeAnnounce66}, - {Name: "TestOldAnnounce66", Fn: s.TestOldAnnounce66}, - {Name: "TestBlockHashAnnounce66", Fn: s.TestBlockHashAnnounce66}, - {Name: "TestMaliciousHandshake66", Fn: s.TestMaliciousHandshake66}, - {Name: "TestMaliciousStatus66", Fn: s.TestMaliciousStatus66}, - {Name: "TestTransaction66", Fn: s.TestTransaction66}, - {Name: "TestMaliciousTx66", Fn: s.TestMaliciousTx66}, - {Name: "TestLargeTxRequest66", Fn: s.TestLargeTxRequest66}, - {Name: "TestNewPooledTxs66", Fn: s.TestNewPooledTxs66}, + {Name: "TestTransaction", Fn: s.TestTransaction}, + {Name: "TestMaliciousTx", Fn: s.TestMaliciousTx}, + {Name: "TestLargeTxRequest", Fn: s.TestLargeTxRequest}, + {Name: "TestNewPooledTxs", Fn: s.TestNewPooledTxs}, } } @@ -135,14 +86,9 @@ func (s *Suite) SnapTests() []utesting.Test { } } -var ( - eth66 = true // indicates whether suite should negotiate eth66 connection - eth65 = false // indicates whether suite should negotiate eth65 connection or below. -) - -// TestStatus65 attempts to connect to the given node and exchange -// a status message with it. -func (s *Suite) TestStatus65(t *utesting.T) { +// TestStatus attempts to connect to the given node and exchange +// a status message with it on the eth protocol. +func (s *Suite) TestStatus(t *utesting.T) { conn, err := s.dial() if err != nil { t.Fatalf("dial failed: %v", err) @@ -153,79 +99,32 @@ func (s *Suite) TestStatus65(t *utesting.T) { } } -// TestStatus66 attempts to connect to the given node and exchange -// a status message with it on the eth66 protocol. -func (s *Suite) TestStatus66(t *utesting.T) { - conn, err := s.dial66() - if err != nil { - t.Fatalf("dial failed: %v", err) - } - defer conn.Close() - if err := conn.peer(s.chain, nil); err != nil { - t.Fatalf("peering failed: %v", err) - } -} - -// TestGetBlockHeaders65 tests whether the given node can respond to -// a `GetBlockHeaders` request accurately. -func (s *Suite) TestGetBlockHeaders65(t *utesting.T) { +// TestGetBlockHeaders tests whether the given node can respond to +// an eth `GetBlockHeaders` request and that the response is accurate. +func (s *Suite) TestGetBlockHeaders(t *utesting.T) { conn, err := s.dial() if err != nil { t.Fatalf("dial failed: %v", err) } defer conn.Close() - if err := conn.peer(s.chain, nil); err != nil { - t.Fatalf("handshake(s) failed: %v", err) - } - // write request - req := &GetBlockHeaders{ - Origin: eth.HashOrNumber{ - Hash: s.chain.blocks[1].Hash(), - }, - Amount: 2, - Skip: 1, - Reverse: false, - } - headers, err := conn.headersRequest(req, s.chain, eth65, 0) - if err != nil { - t.Fatalf("GetBlockHeaders request failed: %v", err) - } - // check for correct headers - expected, err := s.chain.GetHeaders(*req) - if err != nil { - t.Fatalf("failed to get headers for given request: %v", err) - } - if !headersMatch(expected, headers) { - t.Fatalf("header mismatch: \nexpected %v \ngot %v", expected, headers) - } -} - -// TestGetBlockHeaders66 tests whether the given node can respond to -// an eth66 `GetBlockHeaders` request and that the response is accurate. -func (s *Suite) TestGetBlockHeaders66(t *utesting.T) { - conn, err := s.dial66() - if err != nil { - t.Fatalf("dial failed: %v", err) - } - defer conn.Close() if err = conn.peer(s.chain, nil); err != nil { t.Fatalf("peering failed: %v", err) } // write request req := &GetBlockHeaders{ - Origin: eth.HashOrNumber{ - Hash: s.chain.blocks[1].Hash(), + GetBlockHeadersPacket: ð.GetBlockHeadersPacket{ + Origin: eth.HashOrNumber{Hash: s.chain.blocks[1].Hash()}, + Amount: 2, + Skip: 1, + Reverse: false, }, - Amount: 2, - Skip: 1, - Reverse: false, } - headers, err := conn.headersRequest(req, s.chain, eth66, 33) + headers, err := conn.headersRequest(req, s.chain, 33) if err != nil { t.Fatalf("could not get block headers: %v", err) } // check for correct headers - expected, err := s.chain.GetHeaders(*req) + expected, err := s.chain.GetHeaders(req) if err != nil { t.Fatalf("failed to get headers for given request: %v", err) } @@ -234,12 +133,12 @@ func (s *Suite) TestGetBlockHeaders66(t *utesting.T) { } } -// TestSimultaneousRequests66 sends two simultaneous `GetBlockHeader` requests from +// TestSimultaneousRequests sends two simultaneous `GetBlockHeader` requests from // the same connection with different request IDs and checks to make sure the node // responds with the correct headers per request. -func (s *Suite) TestSimultaneousRequests66(t *utesting.T) { +func (s *Suite) TestSimultaneousRequests(t *utesting.T) { // create a connection - conn, err := s.dial66() + conn, err := s.dial() if err != nil { t.Fatalf("dial failed: %v", err) } @@ -247,8 +146,9 @@ func (s *Suite) TestSimultaneousRequests66(t *utesting.T) { if err := conn.peer(s.chain, nil); err != nil { t.Fatalf("peering failed: %v", err) } + // create two requests - req1 := ð.GetBlockHeadersPacket66{ + req1 := &GetBlockHeaders{ RequestId: uint64(111), GetBlockHeadersPacket: ð.GetBlockHeadersPacket{ Origin: eth.HashOrNumber{ @@ -259,7 +159,7 @@ func (s *Suite) TestSimultaneousRequests66(t *utesting.T) { Reverse: false, }, } - req2 := ð.GetBlockHeadersPacket66{ + req2 := &GetBlockHeaders{ RequestId: uint64(222), GetBlockHeadersPacket: ð.GetBlockHeadersPacket{ Origin: eth.HashOrNumber{ @@ -270,46 +170,49 @@ func (s *Suite) TestSimultaneousRequests66(t *utesting.T) { Reverse: false, }, } + // write the first request - if err := conn.Write66(req1, GetBlockHeaders{}.Code()); err != nil { + if err := conn.Write(req1); err != nil { t.Fatalf("failed to write to connection: %v", err) } // write the second request - if err := conn.Write66(req2, GetBlockHeaders{}.Code()); err != nil { + if err := conn.Write(req2); err != nil { t.Fatalf("failed to write to connection: %v", err) } + // wait for responses msg := conn.waitForResponse(s.chain, timeout, req1.RequestId) - headers1, ok := msg.(BlockHeaders) + headers1, ok := msg.(*BlockHeaders) if !ok { t.Fatalf("unexpected %s", pretty.Sdump(msg)) } msg = conn.waitForResponse(s.chain, timeout, req2.RequestId) - headers2, ok := msg.(BlockHeaders) + headers2, ok := msg.(*BlockHeaders) if !ok { t.Fatalf("unexpected %s", pretty.Sdump(msg)) } + // check received headers for accuracy - expected1, err := s.chain.GetHeaders(GetBlockHeaders(*req1.GetBlockHeadersPacket)) + expected1, err := s.chain.GetHeaders(req1) if err != nil { t.Fatalf("failed to get expected headers for request 1: %v", err) } - expected2, err := s.chain.GetHeaders(GetBlockHeaders(*req2.GetBlockHeadersPacket)) + expected2, err := s.chain.GetHeaders(req2) if err != nil { t.Fatalf("failed to get expected headers for request 2: %v", err) } - if !headersMatch(expected1, headers1) { + if !headersMatch(expected1, headers1.BlockHeadersPacket) { t.Fatalf("header mismatch: \nexpected %v \ngot %v", expected1, headers1) } - if !headersMatch(expected2, headers2) { + if !headersMatch(expected2, headers2.BlockHeadersPacket) { t.Fatalf("header mismatch: \nexpected %v \ngot %v", expected2, headers2) } } -// TestSameRequestID66 sends two requests with the same request ID to a +// TestSameRequestID sends two requests with the same request ID to a // single node. -func (s *Suite) TestSameRequestID66(t *utesting.T) { - conn, err := s.dial66() +func (s *Suite) TestSameRequestID(t *utesting.T) { + conn, err := s.dial() if err != nil { t.Fatalf("dial failed: %v", err) } @@ -319,7 +222,7 @@ func (s *Suite) TestSameRequestID66(t *utesting.T) { } // create requests reqID := uint64(1234) - request1 := ð.GetBlockHeadersPacket66{ + request1 := &GetBlockHeaders{ RequestId: reqID, GetBlockHeadersPacket: ð.GetBlockHeadersPacket{ Origin: eth.HashOrNumber{ @@ -328,7 +231,7 @@ func (s *Suite) TestSameRequestID66(t *utesting.T) { Amount: 2, }, } - request2 := ð.GetBlockHeadersPacket66{ + request2 := &GetBlockHeaders{ RequestId: reqID, GetBlockHeadersPacket: ð.GetBlockHeadersPacket{ Origin: eth.HashOrNumber{ @@ -337,45 +240,48 @@ func (s *Suite) TestSameRequestID66(t *utesting.T) { Amount: 2, }, } + // write the requests - if err = conn.Write66(request1, GetBlockHeaders{}.Code()); err != nil { + if err = conn.Write(request1); err != nil { t.Fatalf("failed to write to connection: %v", err) } - if err = conn.Write66(request2, GetBlockHeaders{}.Code()); err != nil { + if err = conn.Write(request2); err != nil { t.Fatalf("failed to write to connection: %v", err) } + // wait for responses msg := conn.waitForResponse(s.chain, timeout, reqID) - headers1, ok := msg.(BlockHeaders) + headers1, ok := msg.(*BlockHeaders) if !ok { t.Fatalf("unexpected %s", pretty.Sdump(msg)) } msg = conn.waitForResponse(s.chain, timeout, reqID) - headers2, ok := msg.(BlockHeaders) + headers2, ok := msg.(*BlockHeaders) if !ok { t.Fatalf("unexpected %s", pretty.Sdump(msg)) } + // check if headers match - expected1, err := s.chain.GetHeaders(GetBlockHeaders(*request1.GetBlockHeadersPacket)) + expected1, err := s.chain.GetHeaders(request1) if err != nil { t.Fatalf("failed to get expected block headers: %v", err) } - expected2, err := s.chain.GetHeaders(GetBlockHeaders(*request2.GetBlockHeadersPacket)) + expected2, err := s.chain.GetHeaders(request2) if err != nil { t.Fatalf("failed to get expected block headers: %v", err) } - if !headersMatch(expected1, headers1) { + if !headersMatch(expected1, headers1.BlockHeadersPacket) { t.Fatalf("header mismatch: \nexpected %v \ngot %v", expected1, headers1) } - if !headersMatch(expected2, headers2) { + if !headersMatch(expected2, headers2.BlockHeadersPacket) { t.Fatalf("header mismatch: \nexpected %v \ngot %v", expected2, headers2) } } -// TestZeroRequestID_66 checks that a message with a request ID of zero is still handled +// TestZeroRequestID checks that a message with a request ID of zero is still handled // by the node. -func (s *Suite) TestZeroRequestID66(t *utesting.T) { - conn, err := s.dial66() +func (s *Suite) TestZeroRequestID(t *utesting.T) { + conn, err := s.dial() if err != nil { t.Fatalf("dial failed: %v", err) } @@ -384,16 +290,16 @@ func (s *Suite) TestZeroRequestID66(t *utesting.T) { t.Fatalf("peering failed: %v", err) } req := &GetBlockHeaders{ - Origin: eth.HashOrNumber{ - Number: 0, + GetBlockHeadersPacket: ð.GetBlockHeadersPacket{ + Origin: eth.HashOrNumber{Number: 0}, + Amount: 2, }, - Amount: 2, } - headers, err := conn.headersRequest(req, s.chain, eth66, 0) + headers, err := conn.headersRequest(req, s.chain, 0) if err != nil { t.Fatalf("failed to get block headers: %v", err) } - expected, err := s.chain.GetHeaders(*req) + expected, err := s.chain.GetHeaders(req) if err != nil { t.Fatalf("failed to get expected block headers: %v", err) } @@ -402,9 +308,9 @@ func (s *Suite) TestZeroRequestID66(t *utesting.T) { } } -// TestGetBlockBodies65 tests whether the given node can respond to +// TestGetBlockBodies tests whether the given node can respond to // a `GetBlockBodies` request and that the response is accurate. -func (s *Suite) TestGetBlockBodies65(t *utesting.T) { +func (s *Suite) TestGetBlockBodies(t *utesting.T) { conn, err := s.dial() if err != nil { t.Fatalf("dial failed: %v", err) @@ -415,126 +321,39 @@ func (s *Suite) TestGetBlockBodies65(t *utesting.T) { } // create block bodies request req := &GetBlockBodies{ - s.chain.blocks[54].Hash(), - s.chain.blocks[75].Hash(), - } - if err := conn.Write(req); err != nil { - t.Fatalf("could not write to connection: %v", err) - } - // wait for response - switch msg := conn.readAndServe(s.chain, timeout).(type) { - case *BlockBodies: - t.Logf("received %d block bodies", len(*msg)) - if len(*msg) != len(*req) { - t.Fatalf("wrong bodies in response: expected %d bodies, "+ - "got %d", len(*req), len(*msg)) - } - default: - t.Fatalf("unexpected: %s", pretty.Sdump(msg)) - } -} - -// TestGetBlockBodies66 tests whether the given node can respond to -// a `GetBlockBodies` request and that the response is accurate over -// the eth66 protocol. -func (s *Suite) TestGetBlockBodies66(t *utesting.T) { - conn, err := s.dial66() - if err != nil { - t.Fatalf("dial failed: %v", err) - } - defer conn.Close() - if err := conn.peer(s.chain, nil); err != nil { - t.Fatalf("peering failed: %v", err) - } - // create block bodies request - req := ð.GetBlockBodiesPacket66{ RequestId: uint64(55), GetBlockBodiesPacket: eth.GetBlockBodiesPacket{ s.chain.blocks[54].Hash(), s.chain.blocks[75].Hash(), }, } - if err := conn.Write66(req, GetBlockBodies{}.Code()); err != nil { + if err := conn.Write(req); err != nil { t.Fatalf("could not write to connection: %v", err) } // wait for block bodies response msg := conn.waitForResponse(s.chain, timeout, req.RequestId) - blockBodies, ok := msg.(BlockBodies) + resp, ok := msg.(*BlockBodies) if !ok { t.Fatalf("unexpected: %s", pretty.Sdump(msg)) } - t.Logf("received %d block bodies", len(blockBodies)) - if len(blockBodies) != len(req.GetBlockBodiesPacket) { + bodies := resp.BlockBodiesPacket + t.Logf("received %d block bodies", len(bodies)) + if len(bodies) != len(req.GetBlockBodiesPacket) { t.Fatalf("wrong bodies in response: expected %d bodies, "+ - "got %d", len(req.GetBlockBodiesPacket), len(blockBodies)) + "got %d", len(req.GetBlockBodiesPacket), len(bodies)) } } -// TestBroadcast65 tests whether a block announcement is correctly -// propagated to the given node's peer(s). -func (s *Suite) TestBroadcast65(t *utesting.T) { - if err := s.sendNextBlock(eth65); err != nil { +// TestBroadcast tests whether a block announcement is correctly +// propagated to the node's peers. +func (s *Suite) TestBroadcast(t *utesting.T) { + if err := s.sendNextBlock(); err != nil { t.Fatalf("block broadcast failed: %v", err) } } -// TestBroadcast66 tests whether a block announcement is correctly -// propagated to the given node's peer(s) on the eth66 protocol. -func (s *Suite) TestBroadcast66(t *utesting.T) { - if err := s.sendNextBlock(eth66); err != nil { - t.Fatalf("block broadcast failed: %v", err) - } -} - -// TestLargeAnnounce65 tests the announcement mechanism with a large block. -func (s *Suite) TestLargeAnnounce65(t *utesting.T) { - nextBlock := len(s.chain.blocks) - blocks := []*NewBlock{ - { - Block: largeBlock(), - TD: s.fullChain.TotalDifficultyAt(nextBlock), - }, - { - Block: s.fullChain.blocks[nextBlock], - TD: largeNumber(2), - }, - { - Block: largeBlock(), - TD: largeNumber(2), - }, - } - - for i, blockAnnouncement := range blocks { - t.Logf("Testing malicious announcement: %v\n", i) - conn, err := s.dial() - if err != nil { - t.Fatalf("dial failed: %v", err) - } - if err = conn.peer(s.chain, nil); err != nil { - t.Fatalf("peering failed: %v", err) - } - if err = conn.Write(blockAnnouncement); err != nil { - t.Fatalf("could not write to connection: %v", err) - } - // Invalid announcement, check that peer disconnected - switch msg := conn.readAndServe(s.chain, time.Second*8).(type) { - case *Disconnect: - case *Error: - break - default: - t.Fatalf("unexpected: %s wanted disconnect", pretty.Sdump(msg)) - } - conn.Close() - } - // Test the last block as a valid block - if err := s.sendNextBlock(eth65); err != nil { - t.Fatalf("failed to broadcast next block: %v", err) - } -} - -// TestLargeAnnounce66 tests the announcement mechanism with a large -// block over the eth66 protocol. -func (s *Suite) TestLargeAnnounce66(t *utesting.T) { +// TestLargeAnnounce tests the announcement mechanism with a large block. +func (s *Suite) TestLargeAnnounce(t *utesting.T) { nextBlock := len(s.chain.blocks) blocks := []*NewBlock{ { @@ -553,7 +372,7 @@ func (s *Suite) TestLargeAnnounce66(t *utesting.T) { for i, blockAnnouncement := range blocks[0:3] { t.Logf("Testing malicious announcement: %v\n", i) - conn, err := s.dial66() + conn, err := s.dial() if err != nil { t.Fatalf("dial failed: %v", err) } @@ -564,7 +383,7 @@ func (s *Suite) TestLargeAnnounce66(t *utesting.T) { t.Fatalf("could not write to connection: %v", err) } // Invalid announcement, check that peer disconnected - switch msg := conn.readAndServe(s.chain, time.Second*8).(type) { + switch msg := conn.readAndServe(s.chain, 8*time.Second).(type) { case *Disconnect: case *Error: break @@ -574,58 +393,35 @@ func (s *Suite) TestLargeAnnounce66(t *utesting.T) { conn.Close() } // Test the last block as a valid block - if err := s.sendNextBlock(eth66); err != nil { + if err := s.sendNextBlock(); err != nil { t.Fatalf("failed to broadcast next block: %v", err) } } -// TestOldAnnounce65 tests the announcement mechanism with an old block. -func (s *Suite) TestOldAnnounce65(t *utesting.T) { - if err := s.oldAnnounce(eth65); err != nil { - t.Fatal(err) - } -} - -// TestOldAnnounce66 tests the announcement mechanism with an old block, -// over the eth66 protocol. -func (s *Suite) TestOldAnnounce66(t *utesting.T) { - if err := s.oldAnnounce(eth66); err != nil { +// TestOldAnnounce tests the announcement mechanism with an old block. +func (s *Suite) TestOldAnnounce(t *utesting.T) { + if err := s.oldAnnounce(); err != nil { t.Fatal(err) } } -// TestBlockHashAnnounce65 sends a new block hash announcement and expects -// the node to perform a `GetBlockHeaders` request. -func (s *Suite) TestBlockHashAnnounce65(t *utesting.T) { - if err := s.hashAnnounce(eth65); err != nil { - t.Fatalf("block hash announcement failed: %v", err) - } -} - -// TestBlockHashAnnounce66 sends a new block hash announcement and expects +// TestBlockHashAnnounce sends a new block hash announcement and expects // the node to perform a `GetBlockHeaders` request. -func (s *Suite) TestBlockHashAnnounce66(t *utesting.T) { - if err := s.hashAnnounce(eth66); err != nil { +func (s *Suite) TestBlockHashAnnounce(t *utesting.T) { + if err := s.hashAnnounce(); err != nil { t.Fatalf("block hash announcement failed: %v", err) } } -// TestMaliciousHandshake65 tries to send malicious data during the handshake. -func (s *Suite) TestMaliciousHandshake65(t *utesting.T) { - if err := s.maliciousHandshakes(t, eth65); err != nil { +// TestMaliciousHandshake tries to send malicious data during the handshake. +func (s *Suite) TestMaliciousHandshake(t *utesting.T) { + if err := s.maliciousHandshakes(t); err != nil { t.Fatal(err) } } -// TestMaliciousHandshake66 tries to send malicious data during the handshake. -func (s *Suite) TestMaliciousHandshake66(t *utesting.T) { - if err := s.maliciousHandshakes(t, eth66); err != nil { - t.Fatal(err) - } -} - -// TestMaliciousStatus65 sends a status package with a large total difficulty. -func (s *Suite) TestMaliciousStatus65(t *utesting.T) { +// TestMaliciousStatus sends a status package with a large total difficulty. +func (s *Suite) TestMaliciousStatus(t *utesting.T) { conn, err := s.dial() if err != nil { t.Fatalf("dial failed: %v", err) @@ -637,58 +433,28 @@ func (s *Suite) TestMaliciousStatus65(t *utesting.T) { } } -// TestMaliciousStatus66 sends a status package with a large total -// difficulty over the eth66 protocol. -func (s *Suite) TestMaliciousStatus66(t *utesting.T) { - conn, err := s.dial66() - if err != nil { - t.Fatalf("dial failed: %v", err) - } - defer conn.Close() - - if err := s.maliciousStatus(conn); err != nil { - t.Fatal(err) - } -} - -// TestTransaction65 sends a valid transaction to the node and -// checks if the transaction gets propagated. -func (s *Suite) TestTransaction65(t *utesting.T) { - if err := s.sendSuccessfulTxs(t, eth65); err != nil { - t.Fatal(err) - } -} - -// TestTransaction66 sends a valid transaction to the node and +// TestTransaction sends a valid transaction to the node and // checks if the transaction gets propagated. -func (s *Suite) TestTransaction66(t *utesting.T) { - if err := s.sendSuccessfulTxs(t, eth66); err != nil { +func (s *Suite) TestTransaction(t *utesting.T) { + if err := s.sendSuccessfulTxs(t); err != nil { t.Fatal(err) } } -// TestMaliciousTx65 sends several invalid transactions and tests whether +// TestMaliciousTx sends several invalid transactions and tests whether // the node will propagate them. -func (s *Suite) TestMaliciousTx65(t *utesting.T) { - if err := s.sendMaliciousTxs(t, eth65); err != nil { +func (s *Suite) TestMaliciousTx(t *utesting.T) { + if err := s.sendMaliciousTxs(t); err != nil { t.Fatal(err) } } -// TestMaliciousTx66 sends several invalid transactions and tests whether -// the node will propagate them. -func (s *Suite) TestMaliciousTx66(t *utesting.T) { - if err := s.sendMaliciousTxs(t, eth66); err != nil { - t.Fatal(err) - } -} - -// TestLargeTxRequest66 tests whether a node can fulfill a large GetPooledTransactions +// TestLargeTxRequest tests whether a node can fulfill a large GetPooledTransactions // request. -func (s *Suite) TestLargeTxRequest66(t *utesting.T) { +func (s *Suite) TestLargeTxRequest(t *utesting.T) { // send the next block to ensure the node is no longer syncing and // is able to accept txs - if err := s.sendNextBlock(eth66); err != nil { + if err := s.sendNextBlock(); err != nil { t.Fatalf("failed to send next block: %v", err) } // send 2000 transactions to the node @@ -701,7 +467,7 @@ func (s *Suite) TestLargeTxRequest66(t *utesting.T) { } // set up connection to receive to ensure node is peered with the receiving connection // before tx request is sent - conn, err := s.dial66() + conn, err := s.dial() if err != nil { t.Fatalf("dial failed: %v", err) } @@ -714,17 +480,17 @@ func (s *Suite) TestLargeTxRequest66(t *utesting.T) { for _, hash := range hashMap { hashes = append(hashes, hash) } - getTxReq := ð.GetPooledTransactionsPacket66{ + getTxReq := &GetPooledTransactions{ RequestId: 1234, GetPooledTransactionsPacket: hashes, } - if err = conn.Write66(getTxReq, GetPooledTransactions{}.Code()); err != nil { + if err = conn.Write(getTxReq); err != nil { t.Fatalf("could not write to conn: %v", err) } // check that all received transactions match those that were sent to node switch msg := conn.waitForResponse(s.chain, timeout, getTxReq.RequestId).(type) { - case PooledTransactions: - for _, gotTx := range msg { + case *PooledTransactions: + for _, gotTx := range msg.PooledTransactionsPacket { if _, exists := hashMap[gotTx.Hash()]; !exists { t.Fatalf("unexpected tx received: %v", gotTx.Hash()) } @@ -734,12 +500,12 @@ func (s *Suite) TestLargeTxRequest66(t *utesting.T) { } } -// TestNewPooledTxs_66 tests whether a node will do a GetPooledTransactions +// TestNewPooledTxs tests whether a node will do a GetPooledTransactions // request upon receiving a NewPooledTransactionHashes announcement. -func (s *Suite) TestNewPooledTxs66(t *utesting.T) { +func (s *Suite) TestNewPooledTxs(t *utesting.T) { // send the next block to ensure the node is no longer syncing and // is able to accept txs - if err := s.sendNextBlock(eth66); err != nil { + if err := s.sendNextBlock(); err != nil { t.Fatalf("failed to send next block: %v", err) } @@ -757,7 +523,7 @@ func (s *Suite) TestNewPooledTxs66(t *utesting.T) { announce := NewPooledTransactionHashes(hashes) // send announcement - conn, err := s.dial66() + conn, err := s.dial() if err != nil { t.Fatalf("dial failed: %v", err) } @@ -771,16 +537,20 @@ func (s *Suite) TestNewPooledTxs66(t *utesting.T) { // wait for GetPooledTxs request for { - _, msg := conn.readAndServe66(s.chain, timeout) + msg := conn.readAndServe(s.chain, timeout) switch msg := msg.(type) { - case GetPooledTransactions: - if len(msg) != len(hashes) { - t.Fatalf("unexpected number of txs requested: wanted %d, got %d", len(hashes), len(msg)) + case *GetPooledTransactions: + if len(msg.GetPooledTransactionsPacket) != len(hashes) { + t.Fatalf("unexpected number of txs requested: wanted %d, got %d", len(hashes), len(msg.GetPooledTransactionsPacket)) } return + // ignore propagated txs from previous tests case *NewPooledTransactionHashes: continue + case *Transactions: + continue + // ignore block announcements from previous tests case *NewBlockHashes: continue diff --git a/cmd/devp2p/internal/ethtest/suite_test.go b/cmd/devp2p/internal/ethtest/suite_test.go index 924c80d01c8cb..8a2b132fa3b15 100644 --- a/cmd/devp2p/internal/ethtest/suite_test.go +++ b/cmd/devp2p/internal/ethtest/suite_test.go @@ -45,7 +45,7 @@ func TestEthSuite(t *testing.T) { if err != nil { t.Fatalf("could not create new test suite: %v", err) } - for _, test := range suite.Eth66Tests() { + for _, test := range suite.EthTests() { t.Run(test.Name, func(t *testing.T) { result := utesting.RunTAP([]utesting.Test{{Name: test.Name, Fn: test.Fn}}, os.Stdout) if result[0].Failed { diff --git a/cmd/devp2p/internal/ethtest/transaction.go b/cmd/devp2p/internal/ethtest/transaction.go index 5d722f417a223..baa55bd49268d 100644 --- a/cmd/devp2p/internal/ethtest/transaction.go +++ b/cmd/devp2p/internal/ethtest/transaction.go @@ -29,10 +29,10 @@ import ( "github.com/ethereum/go-ethereum/params" ) -//var faucetAddr = common.HexToAddress("0x71562b71999873DB5b286dF957af199Ec94617F7") +// var faucetAddr = common.HexToAddress("0x71562b71999873DB5b286dF957af199Ec94617F7") var faucetKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") -func (s *Suite) sendSuccessfulTxs(t *utesting.T, isEth66 bool) error { +func (s *Suite) sendSuccessfulTxs(t *utesting.T) error { tests := []*types.Transaction{ getNextTxFromChain(s), unknownTx(s), @@ -48,15 +48,15 @@ func (s *Suite) sendSuccessfulTxs(t *utesting.T, isEth66 bool) error { prevTx = tests[i-1] } // write tx to connection - if err := sendSuccessfulTx(s, tx, prevTx, isEth66); err != nil { + if err := sendSuccessfulTx(s, tx, prevTx); err != nil { return fmt.Errorf("send successful tx test failed: %v", err) } } return nil } -func sendSuccessfulTx(s *Suite, tx *types.Transaction, prevTx *types.Transaction, isEth66 bool) error { - sendConn, recvConn, err := s.createSendAndRecvConns(isEth66) +func sendSuccessfulTx(s *Suite, tx *types.Transaction, prevTx *types.Transaction) error { + sendConn, recvConn, err := s.createSendAndRecvConns() if err != nil { return err } @@ -73,8 +73,10 @@ func sendSuccessfulTx(s *Suite, tx *types.Transaction, prevTx *types.Transaction if err = recvConn.peer(s.chain, nil); err != nil { return fmt.Errorf("peering failed: %v", err) } + // update last nonce seen nonce = tx.Nonce() + // Wait for the transaction announcement for { switch msg := recvConn.readAndServe(s.chain, timeout).(type) { @@ -114,7 +116,7 @@ func sendSuccessfulTx(s *Suite, tx *types.Transaction, prevTx *types.Transaction } } -func (s *Suite) sendMaliciousTxs(t *utesting.T, isEth66 bool) error { +func (s *Suite) sendMaliciousTxs(t *utesting.T) error { badTxs := []*types.Transaction{ getOldTxFromChain(s), invalidNonceTx(s), @@ -122,16 +124,9 @@ func (s *Suite) sendMaliciousTxs(t *utesting.T, isEth66 bool) error { hugeGasPrice(s), hugeData(s), } + // setup receiving connection before sending malicious txs - var ( - recvConn *Conn - err error - ) - if isEth66 { - recvConn, err = s.dial66() - } else { - recvConn, err = s.dial() - } + recvConn, err := s.dial() if err != nil { return fmt.Errorf("dial failed: %v", err) } @@ -139,9 +134,10 @@ func (s *Suite) sendMaliciousTxs(t *utesting.T, isEth66 bool) error { if err = recvConn.peer(s.chain, nil); err != nil { return fmt.Errorf("peering failed: %v", err) } + for i, tx := range badTxs { t.Logf("Testing malicious tx propagation: %v\n", i) - if err = sendMaliciousTx(s, tx, isEth66); err != nil { + if err = sendMaliciousTx(s, tx); err != nil { return fmt.Errorf("malicious tx test failed:\ntx: %v\nerror: %v", tx, err) } } @@ -149,17 +145,8 @@ func (s *Suite) sendMaliciousTxs(t *utesting.T, isEth66 bool) error { return checkMaliciousTxPropagation(s, badTxs, recvConn) } -func sendMaliciousTx(s *Suite, tx *types.Transaction, isEth66 bool) error { - // setup connection - var ( - conn *Conn - err error - ) - if isEth66 { - conn, err = s.dial66() - } else { - conn, err = s.dial() - } +func sendMaliciousTx(s *Suite, tx *types.Transaction) error { + conn, err := s.dial() if err != nil { return fmt.Errorf("dial failed: %v", err) } @@ -167,6 +154,7 @@ func sendMaliciousTx(s *Suite, tx *types.Transaction, isEth66 bool) error { if err = conn.peer(s.chain, nil); err != nil { return fmt.Errorf("peering failed: %v", err) } + // write malicious tx if err = conn.Write(&Transactions{tx}); err != nil { return fmt.Errorf("failed to write to connection: %v", err) @@ -182,7 +170,7 @@ func sendMultipleSuccessfulTxs(t *utesting.T, s *Suite, txs []*types.Transaction txMsg := Transactions(txs) t.Logf("sending %d txs\n", len(txs)) - sendConn, recvConn, err := s.createSendAndRecvConns(true) + sendConn, recvConn, err := s.createSendAndRecvConns() if err != nil { return err } @@ -194,16 +182,20 @@ func sendMultipleSuccessfulTxs(t *utesting.T, s *Suite, txs []*types.Transaction if err = recvConn.peer(s.chain, nil); err != nil { return fmt.Errorf("peering failed: %v", err) } + // Send the transactions if err = sendConn.Write(&txMsg); err != nil { return fmt.Errorf("failed to write message to connection: %v", err) } + // update nonce nonce = txs[len(txs)-1].Nonce() - // Wait for the transaction announcement(s) and make sure all sent txs are being propagated + + // Wait for the transaction announcement(s) and make sure all sent txs are being propagated. + // all txs should be announced within a couple announcements. recvHashes := make([]common.Hash, 0) - // all txs should be announced within 3 announcements - for i := 0; i < 3; i++ { + + for i := 0; i < 20; i++ { switch msg := recvConn.readAndServe(s.chain, timeout).(type) { case *Transactions: for _, tx := range *msg { diff --git a/cmd/devp2p/internal/ethtest/types.go b/cmd/devp2p/internal/ethtest/types.go index e92b543940679..2c5cb94c699f1 100644 --- a/cmd/devp2p/internal/ethtest/types.go +++ b/cmd/devp2p/internal/ethtest/types.go @@ -29,6 +29,7 @@ import ( type Message interface { Code() int + ReqID() uint64 } type Error struct { @@ -37,9 +38,11 @@ type Error struct { func (e *Error) Unwrap() error { return e.err } func (e *Error) Error() string { return e.err.Error() } -func (e *Error) Code() int { return -1 } func (e *Error) String() string { return e.Error() } +func (e *Error) Code() int { return -1 } +func (e *Error) ReqID() uint64 { return 0 } + func errorf(format string, args ...interface{}) *Error { return &Error{fmt.Errorf(format, args...)} } @@ -56,73 +59,88 @@ type Hello struct { Rest []rlp.RawValue `rlp:"tail"` } -func (h Hello) Code() int { return 0x00 } +func (msg Hello) Code() int { return 0x00 } +func (msg Hello) ReqID() uint64 { return 0 } // Disconnect is the RLP structure for a disconnect message. type Disconnect struct { Reason p2p.DiscReason } -func (d Disconnect) Code() int { return 0x01 } +func (msg Disconnect) Code() int { return 0x01 } +func (msg Disconnect) ReqID() uint64 { return 0 } type Ping struct{} -func (p Ping) Code() int { return 0x02 } +func (msg Ping) Code() int { return 0x02 } +func (msg Ping) ReqID() uint64 { return 0 } type Pong struct{} -func (p Pong) Code() int { return 0x03 } +func (msg Pong) Code() int { return 0x03 } +func (msg Pong) ReqID() uint64 { return 0 } // Status is the network packet for the status message for eth/64 and later. type Status eth.StatusPacket -func (s Status) Code() int { return 16 } +func (msg Status) Code() int { return 16 } +func (msg Status) ReqID() uint64 { return 0 } // NewBlockHashes is the network packet for the block announcements. type NewBlockHashes eth.NewBlockHashesPacket -func (nbh NewBlockHashes) Code() int { return 17 } +func (msg NewBlockHashes) Code() int { return 17 } +func (msg NewBlockHashes) ReqID() uint64 { return 0 } type Transactions eth.TransactionsPacket -func (t Transactions) Code() int { return 18 } +func (msg Transactions) Code() int { return 18 } +func (msg Transactions) ReqID() uint64 { return 18 } // GetBlockHeaders represents a block header query. -type GetBlockHeaders eth.GetBlockHeadersPacket +type GetBlockHeaders eth.GetBlockHeadersPacket66 -func (g GetBlockHeaders) Code() int { return 19 } +func (msg GetBlockHeaders) Code() int { return 19 } +func (msg GetBlockHeaders) ReqID() uint64 { return msg.RequestId } -type BlockHeaders eth.BlockHeadersPacket +type BlockHeaders eth.BlockHeadersPacket66 -func (bh BlockHeaders) Code() int { return 20 } +func (msg BlockHeaders) Code() int { return 20 } +func (msg BlockHeaders) ReqID() uint64 { return msg.RequestId } // GetBlockBodies represents a GetBlockBodies request -type GetBlockBodies eth.GetBlockBodiesPacket +type GetBlockBodies eth.GetBlockBodiesPacket66 -func (gbb GetBlockBodies) Code() int { return 21 } +func (msg GetBlockBodies) Code() int { return 21 } +func (msg GetBlockBodies) ReqID() uint64 { return msg.RequestId } // BlockBodies is the network packet for block content distribution. -type BlockBodies eth.BlockBodiesPacket +type BlockBodies eth.BlockBodiesPacket66 -func (bb BlockBodies) Code() int { return 22 } +func (msg BlockBodies) Code() int { return 22 } +func (msg BlockBodies) ReqID() uint64 { return msg.RequestId } // NewBlock is the network packet for the block propagation message. type NewBlock eth.NewBlockPacket -func (nb NewBlock) Code() int { return 23 } +func (msg NewBlock) Code() int { return 23 } +func (msg NewBlock) ReqID() uint64 { return 0 } // NewPooledTransactionHashes is the network packet for the tx hash propagation message. type NewPooledTransactionHashes eth.NewPooledTransactionHashesPacket -func (nb NewPooledTransactionHashes) Code() int { return 24 } +func (msg NewPooledTransactionHashes) Code() int { return 24 } +func (msg NewPooledTransactionHashes) ReqID() uint64 { return 0 } -type GetPooledTransactions eth.GetPooledTransactionsPacket +type GetPooledTransactions eth.GetPooledTransactionsPacket66 -func (gpt GetPooledTransactions) Code() int { return 25 } +func (msg GetPooledTransactions) Code() int { return 25 } +func (msg GetPooledTransactions) ReqID() uint64 { return msg.RequestId } -type PooledTransactions eth.PooledTransactionsPacket +type PooledTransactions eth.PooledTransactionsPacket66 -func (pt PooledTransactions) Code() int { return 26 } +func (msg PooledTransactions) Code() int { return 26 } +func (msg PooledTransactions) ReqID() uint64 { return msg.RequestId } // Conn represents an individual connection with a peer type Conn struct { @@ -135,62 +153,13 @@ type Conn struct { caps []p2p.Cap } -// Read reads an eth packet from the connection. +// Read reads an eth66 packet from the connection. func (c *Conn) Read() Message { code, rawData, _, err := c.Conn.Read() if err != nil { return errorf("could not read from connection: %v", err) } - var msg Message - switch int(code) { - case (Hello{}).Code(): - msg = new(Hello) - case (Ping{}).Code(): - msg = new(Ping) - case (Pong{}).Code(): - msg = new(Pong) - case (Disconnect{}).Code(): - msg = new(Disconnect) - case (Status{}).Code(): - msg = new(Status) - case (GetBlockHeaders{}).Code(): - msg = new(GetBlockHeaders) - case (BlockHeaders{}).Code(): - msg = new(BlockHeaders) - case (GetBlockBodies{}).Code(): - msg = new(GetBlockBodies) - case (BlockBodies{}).Code(): - msg = new(BlockBodies) - case (NewBlock{}).Code(): - msg = new(NewBlock) - case (NewBlockHashes{}).Code(): - msg = new(NewBlockHashes) - case (Transactions{}).Code(): - msg = new(Transactions) - case (NewPooledTransactionHashes{}).Code(): - msg = new(NewPooledTransactionHashes) - case (GetPooledTransactions{}.Code()): - msg = new(GetPooledTransactions) - case (PooledTransactions{}.Code()): - msg = new(PooledTransactions) - default: - return errorf("invalid message code: %d", code) - } - // if message is devp2p, decode here - if err := rlp.DecodeBytes(rawData, msg); err != nil { - return errorf("could not rlp decode message: %v", err) - } - return msg -} - -// Read66 reads an eth66 packet from the connection. -func (c *Conn) Read66() (uint64, Message) { - code, rawData, _, err := c.Conn.Read() - if err != nil { - return 0, errorf("could not read from connection: %v", err) - } - var msg Message switch int(code) { case (Hello{}).Code(): @@ -206,27 +175,27 @@ func (c *Conn) Read66() (uint64, Message) { case (GetBlockHeaders{}).Code(): ethMsg := new(eth.GetBlockHeadersPacket66) if err := rlp.DecodeBytes(rawData, ethMsg); err != nil { - return 0, errorf("could not rlp decode message: %v", err) + return errorf("could not rlp decode message: %v", err) } - return ethMsg.RequestId, GetBlockHeaders(*ethMsg.GetBlockHeadersPacket) + return (*GetBlockHeaders)(ethMsg) case (BlockHeaders{}).Code(): ethMsg := new(eth.BlockHeadersPacket66) if err := rlp.DecodeBytes(rawData, ethMsg); err != nil { - return 0, errorf("could not rlp decode message: %v", err) + return errorf("could not rlp decode message: %v", err) } - return ethMsg.RequestId, BlockHeaders(ethMsg.BlockHeadersPacket) + return (*BlockHeaders)(ethMsg) case (GetBlockBodies{}).Code(): ethMsg := new(eth.GetBlockBodiesPacket66) if err := rlp.DecodeBytes(rawData, ethMsg); err != nil { - return 0, errorf("could not rlp decode message: %v", err) + return errorf("could not rlp decode message: %v", err) } - return ethMsg.RequestId, GetBlockBodies(ethMsg.GetBlockBodiesPacket) + return (*GetBlockBodies)(ethMsg) case (BlockBodies{}).Code(): ethMsg := new(eth.BlockBodiesPacket66) if err := rlp.DecodeBytes(rawData, ethMsg); err != nil { - return 0, errorf("could not rlp decode message: %v", err) + return errorf("could not rlp decode message: %v", err) } - return ethMsg.RequestId, BlockBodies(ethMsg.BlockBodiesPacket) + return (*BlockBodies)(ethMsg) case (NewBlock{}).Code(): msg = new(NewBlock) case (NewBlockHashes{}).Code(): @@ -238,26 +207,26 @@ func (c *Conn) Read66() (uint64, Message) { case (GetPooledTransactions{}.Code()): ethMsg := new(eth.GetPooledTransactionsPacket66) if err := rlp.DecodeBytes(rawData, ethMsg); err != nil { - return 0, errorf("could not rlp decode message: %v", err) + return errorf("could not rlp decode message: %v", err) } - return ethMsg.RequestId, GetPooledTransactions(ethMsg.GetPooledTransactionsPacket) + return (*GetPooledTransactions)(ethMsg) case (PooledTransactions{}.Code()): ethMsg := new(eth.PooledTransactionsPacket66) if err := rlp.DecodeBytes(rawData, ethMsg); err != nil { - return 0, errorf("could not rlp decode message: %v", err) + return errorf("could not rlp decode message: %v", err) } - return ethMsg.RequestId, PooledTransactions(ethMsg.PooledTransactionsPacket) + return (*PooledTransactions)(ethMsg) default: msg = errorf("invalid message code: %d", code) } if msg != nil { if err := rlp.DecodeBytes(rawData, msg); err != nil { - return 0, errorf("could not rlp decode message: %v", err) + return errorf("could not rlp decode message: %v", err) } - return 0, msg + return msg } - return 0, errorf("invalid message: %s", string(rawData)) + return errorf("invalid message: %s", string(rawData)) } // Write writes a eth packet to the connection. @@ -270,16 +239,6 @@ func (c *Conn) Write(msg Message) error { return err } -// Write66 writes an eth66 packet to the connection. -func (c *Conn) Write66(req eth.Packet, code int) error { - payload, err := rlp.EncodeToBytes(req) - if err != nil { - return err - } - _, err = c.Conn.Write(uint64(code), payload) - return err -} - // ReadSnap reads a snap/1 response with the given id from the connection. func (c *Conn) ReadSnap(id uint64) (Message, error) { respId := id + 1 @@ -315,7 +274,6 @@ func (c *Conn) ReadSnap(id uint64) (Message, error) { return nil, fmt.Errorf("could not rlp decode message: %v", err) } return snpMsg.(Message), nil - } return nil, fmt.Errorf("request timed out") } diff --git a/cmd/devp2p/internal/v5test/framework.go b/cmd/devp2p/internal/v5test/framework.go index 9eac37520f7b7..6ccbbd075bf0f 100644 --- a/cmd/devp2p/internal/v5test/framework.go +++ b/cmd/devp2p/internal/v5test/framework.go @@ -60,11 +60,9 @@ type conn struct { remoteAddr *net.UDPAddr listeners []net.PacketConn - log logger - codec *v5wire.Codec - lastRequest v5wire.Packet - lastChallenge *v5wire.Whoareyou - idCounter uint32 + log logger + codec *v5wire.Codec + idCounter uint32 } type logger interface { diff --git a/cmd/devp2p/keycmd.go b/cmd/devp2p/keycmd.go index 869b8c2a44f0f..e824abe653e2b 100644 --- a/cmd/devp2p/keycmd.go +++ b/cmd/devp2p/keycmd.go @@ -22,25 +22,25 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/p2p/enode" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) var ( - keyCommand = cli.Command{ + keyCommand = &cli.Command{ Name: "key", Usage: "Operations on node keys", - Subcommands: []cli.Command{ + Subcommands: []*cli.Command{ keyGenerateCommand, keyToNodeCommand, }, } - keyGenerateCommand = cli.Command{ + keyGenerateCommand = &cli.Command{ Name: "generate", Usage: "Generates node key files", ArgsUsage: "keyfile", Action: genkey, } - keyToNodeCommand = cli.Command{ + keyToNodeCommand = &cli.Command{ Name: "to-enode", Usage: "Creates an enode URL from a node key file", ArgsUsage: "keyfile", @@ -50,17 +50,17 @@ var ( ) var ( - hostFlag = cli.StringFlag{ + hostFlag = &cli.StringFlag{ Name: "ip", Usage: "IP address of the node", Value: "127.0.0.1", } - tcpPortFlag = cli.IntFlag{ + tcpPortFlag = &cli.IntFlag{ Name: "tcp", Usage: "TCP port of the node", Value: 30303, } - udpPortFlag = cli.IntFlag{ + udpPortFlag = &cli.IntFlag{ Name: "udp", Usage: "UDP port of the node", Value: 30303, diff --git a/cmd/devp2p/main.go b/cmd/devp2p/main.go index 4a4e905a424ec..9e13d29ab72d6 100644 --- a/cmd/devp2p/main.go +++ b/cmd/devp2p/main.go @@ -19,32 +19,20 @@ package main import ( "fmt" "os" - "path/filepath" - "sort" "github.com/ethereum/go-ethereum/internal/debug" + "github.com/ethereum/go-ethereum/internal/flags" "github.com/ethereum/go-ethereum/p2p/enode" - "github.com/ethereum/go-ethereum/params" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) -var ( - // Git information set by linker when building with ci.go. - gitCommit string - gitDate string - app = &cli.App{ - Name: filepath.Base(os.Args[0]), - Usage: "go-ethereum devp2p tool", - Version: params.VersionWithCommit(gitCommit, gitDate), - Writer: os.Stdout, - HideVersion: true, - } -) +var app = flags.NewApp("go-ethereum devp2p tool") func init() { - // Set up the CLI app. + app.HideVersion = true app.Flags = append(app.Flags, debug.Flags...) app.Before = func(ctx *cli.Context) error { + flags.MigrateGlobalFlags(ctx) return debug.Setup(ctx) } app.After = func(ctx *cli.Context) error { @@ -55,8 +43,9 @@ func init() { fmt.Fprintf(os.Stderr, "No such command: %s\n", cmd) os.Exit(1) } + // Add subcommands. - app.Commands = []cli.Command{ + app.Commands = []*cli.Command{ enrdumpCommand, keyCommand, discv4Command, @@ -73,10 +62,17 @@ func main() { // commandHasFlag returns true if the current command supports the given flag. func commandHasFlag(ctx *cli.Context, flag cli.Flag) bool { - flags := ctx.FlagNames() - sort.Strings(flags) - i := sort.SearchStrings(flags, flag.GetName()) - return i != len(flags) && flags[i] == flag.GetName() + names := flag.Names() + set := make(map[string]struct{}, len(names)) + for _, name := range names { + set[name] = struct{}{} + } + for _, fn := range ctx.FlagNames() { + if _, ok := set[fn]; ok { + return true + } + } + return false } // getNodeArg handles the common case of a single node descriptor argument. @@ -84,7 +80,7 @@ func getNodeArg(ctx *cli.Context) *enode.Node { if ctx.NArg() < 1 { exit("missing node as command-line argument") } - n, err := parseNode(ctx.Args()[0]) + n, err := parseNode(ctx.Args().First()) if err != nil { exit(err) } diff --git a/cmd/devp2p/nodesetcmd.go b/cmd/devp2p/nodesetcmd.go index d65d6314c8e13..2cf1045928346 100644 --- a/cmd/devp2p/nodesetcmd.go +++ b/cmd/devp2p/nodesetcmd.go @@ -29,25 +29,25 @@ import ( "github.com/ethereum/go-ethereum/p2p/enr" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) var ( - nodesetCommand = cli.Command{ + nodesetCommand = &cli.Command{ Name: "nodeset", Usage: "Node set tools", - Subcommands: []cli.Command{ + Subcommands: []*cli.Command{ nodesetInfoCommand, nodesetFilterCommand, }, } - nodesetInfoCommand = cli.Command{ + nodesetInfoCommand = &cli.Command{ Name: "info", Usage: "Shows statistics about a node set", Action: nodesetInfo, ArgsUsage: "", } - nodesetFilterCommand = cli.Command{ + nodesetFilterCommand = &cli.Command{ Name: "filter", Usage: "Filters a node set", Action: nodesetFilter, @@ -181,7 +181,7 @@ func parseFilterLimit(args []string) (int, error) { return limit, nil } -// andFilter parses node filters in args and and returns a single filter that requires all +// andFilter parses node filters in args and returns a single filter that requires all // of them to match. func andFilter(args []string) (nodeFilter, error) { checks, err := parseFilters(args) diff --git a/cmd/devp2p/rlpxcmd.go b/cmd/devp2p/rlpxcmd.go index 6557a239da771..42b38120c475a 100644 --- a/cmd/devp2p/rlpxcmd.go +++ b/cmd/devp2p/rlpxcmd.go @@ -22,29 +22,28 @@ import ( "github.com/ethereum/go-ethereum/cmd/devp2p/internal/ethtest" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/internal/utesting" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/rlpx" "github.com/ethereum/go-ethereum/rlp" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) var ( - rlpxCommand = cli.Command{ + rlpxCommand = &cli.Command{ Name: "rlpx", Usage: "RLPx Commands", - Subcommands: []cli.Command{ + Subcommands: []*cli.Command{ rlpxPingCommand, rlpxEthTestCommand, rlpxSnapTestCommand, }, } - rlpxPingCommand = cli.Command{ + rlpxPingCommand = &cli.Command{ Name: "ping", Usage: "ping ", Action: rlpxPing, } - rlpxEthTestCommand = cli.Command{ + rlpxEthTestCommand = &cli.Command{ Name: "eth-test", Usage: "Runs tests against a node", ArgsUsage: " ", @@ -54,7 +53,7 @@ var ( testTAPFlag, }, } - rlpxSnapTestCommand = cli.Command{ + rlpxSnapTestCommand = &cli.Command{ Name: "snap-test", Usage: "Runs tests against a node", ArgsUsage: " ", @@ -106,16 +105,11 @@ func rlpxEthTest(ctx *cli.Context) error { if ctx.NArg() < 3 { exit("missing path to chain.rlp as command-line argument") } - suite, err := ethtest.NewSuite(getNodeArg(ctx), ctx.Args()[1], ctx.Args()[2]) + suite, err := ethtest.NewSuite(getNodeArg(ctx), ctx.Args().Get(1), ctx.Args().Get(2)) if err != nil { exit(err) } - // check if given node supports eth66, and if so, run eth66 protocol tests as well - is66Failed, _ := utesting.Run(utesting.Test{Name: "Is_66", Fn: suite.Is_66}) - if is66Failed { - return runTests(ctx, suite.EthTests()) - } - return runTests(ctx, suite.AllEthTests()) + return runTests(ctx, suite.EthTests()) } // rlpxSnapTest runs the snap protocol test suite. @@ -123,7 +117,7 @@ func rlpxSnapTest(ctx *cli.Context) error { if ctx.NArg() < 3 { exit("missing path to chain.rlp as command-line argument") } - suite, err := ethtest.NewSuite(getNodeArg(ctx), ctx.Args()[1], ctx.Args()[2]) + suite, err := ethtest.NewSuite(getNodeArg(ctx), ctx.Args().Get(1), ctx.Args().Get(2)) if err != nil { exit(err) } diff --git a/cmd/devp2p/runtest.go b/cmd/devp2p/runtest.go index 4168f8555bfbd..f72aa91119c5f 100644 --- a/cmd/devp2p/runtest.go +++ b/cmd/devp2p/runtest.go @@ -22,25 +22,25 @@ import ( "github.com/ethereum/go-ethereum/cmd/devp2p/internal/v4test" "github.com/ethereum/go-ethereum/internal/utesting" "github.com/ethereum/go-ethereum/log" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) var ( - testPatternFlag = cli.StringFlag{ + testPatternFlag = &cli.StringFlag{ Name: "run", Usage: "Pattern of test suite(s) to run", } - testTAPFlag = cli.BoolFlag{ + testTAPFlag = &cli.BoolFlag{ Name: "tap", Usage: "Output TAP", } // These two are specific to the discovery tests. - testListen1Flag = cli.StringFlag{ + testListen1Flag = &cli.StringFlag{ Name: "listen1", Usage: "IP address of the first tester", Value: v4test.Listen1, } - testListen2Flag = cli.StringFlag{ + testListen2Flag = &cli.StringFlag{ Name: "listen2", Usage: "IP address of the second tester", Value: v4test.Listen2, @@ -53,7 +53,7 @@ func runTests(ctx *cli.Context, tests []utesting.Test) error { tests = utesting.MatchTests(tests, ctx.String(testPatternFlag.Name)) } // Disable logging unless explicitly enabled. - if !ctx.GlobalIsSet("verbosity") && !ctx.GlobalIsSet("vmodule") { + if !ctx.IsSet("verbosity") && !ctx.IsSet("vmodule") { log.Root().SetHandler(log.DiscardHandler()) } // Run the tests. diff --git a/cmd/ethkey/changepassword.go b/cmd/ethkey/changepassword.go index bd8745f6db87b..4298e2b834079 100644 --- a/cmd/ethkey/changepassword.go +++ b/cmd/ethkey/changepassword.go @@ -23,15 +23,15 @@ import ( "github.com/ethereum/go-ethereum/accounts/keystore" "github.com/ethereum/go-ethereum/cmd/utils" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) -var newPassphraseFlag = cli.StringFlag{ +var newPassphraseFlag = &cli.StringFlag{ Name: "newpasswordfile", Usage: "the file that contains the new password for the keyfile", } -var commandChangePassphrase = cli.Command{ +var commandChangePassphrase = &cli.Command{ Name: "changepassword", Usage: "change the password on a keyfile", ArgsUsage: "", diff --git a/cmd/ethkey/generate.go b/cmd/ethkey/generate.go index 1b70b130bcd54..60d8b3c7795bf 100644 --- a/cmd/ethkey/generate.go +++ b/cmd/ethkey/generate.go @@ -26,7 +26,7 @@ import ( "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/crypto" "github.com/google/uuid" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) type outputGenerate struct { @@ -35,17 +35,17 @@ type outputGenerate struct { } var ( - privateKeyFlag = cli.StringFlag{ + privateKeyFlag = &cli.StringFlag{ Name: "privatekey", Usage: "file containing a raw private key to encrypt", } - lightKDFFlag = cli.BoolFlag{ + lightKDFFlag = &cli.BoolFlag{ Name: "lightkdf", Usage: "use less secure scrypt parameters", } ) -var commandGenerate = cli.Command{ +var commandGenerate = &cli.Command{ Name: "generate", Usage: "generate new keyfile", ArgsUsage: "[ ]", diff --git a/cmd/ethkey/inspect.go b/cmd/ethkey/inspect.go index efcaecd389d3b..29b1c13e859b9 100644 --- a/cmd/ethkey/inspect.go +++ b/cmd/ethkey/inspect.go @@ -24,7 +24,7 @@ import ( "github.com/ethereum/go-ethereum/accounts/keystore" "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/crypto" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) type outputInspect struct { @@ -34,13 +34,13 @@ type outputInspect struct { } var ( - privateFlag = cli.BoolFlag{ + privateFlag = &cli.BoolFlag{ Name: "private", Usage: "include the private key in the output", } ) -var commandInspect = cli.Command{ +var commandInspect = &cli.Command{ Name: "inspect", Usage: "inspect a keyfile", ArgsUsage: "", diff --git a/cmd/ethkey/main.go b/cmd/ethkey/main.go index 6db39174c4615..25c0d104f61e9 100644 --- a/cmd/ethkey/main.go +++ b/cmd/ethkey/main.go @@ -21,38 +21,33 @@ import ( "os" "github.com/ethereum/go-ethereum/internal/flags" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) const ( defaultKeyfileName = "keyfile.json" ) -// Git SHA1 commit hash of the release (set via linker flags) -var gitCommit = "" -var gitDate = "" - var app *cli.App func init() { - app = flags.NewApp(gitCommit, gitDate, "an Ethereum key manager") - app.Commands = []cli.Command{ + app = flags.NewApp("Ethereum key manager") + app.Commands = []*cli.Command{ commandGenerate, commandInspect, commandChangePassphrase, commandSignMessage, commandVerifyMessage, } - cli.CommandHelpTemplate = flags.OriginCommandHelpTemplate } // Commonly used command line flags. var ( - passphraseFlag = cli.StringFlag{ + passphraseFlag = &cli.StringFlag{ Name: "passwordfile", Usage: "the file that contains the password for the keyfile", } - jsonFlag = cli.BoolFlag{ + jsonFlag = &cli.BoolFlag{ Name: "json", Usage: "output JSON instead of human-readable format", } diff --git a/cmd/ethkey/message.go b/cmd/ethkey/message.go index 1a58eeb536fa5..6b8dec03cd670 100644 --- a/cmd/ethkey/message.go +++ b/cmd/ethkey/message.go @@ -21,23 +21,24 @@ import ( "fmt" "os" + "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/accounts/keystore" "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) type outputSign struct { Signature string } -var msgfileFlag = cli.StringFlag{ +var msgfileFlag = &cli.StringFlag{ Name: "msgfile", Usage: "file containing the message to sign/verify", } -var commandSignMessage = cli.Command{ +var commandSignMessage = &cli.Command{ Name: "signmessage", Usage: "sign a message", ArgsUsage: " ", @@ -68,7 +69,7 @@ To sign a message contained in a file, use the --msgfile flag. utils.Fatalf("Error decrypting key: %v", err) } - signature, err := crypto.Sign(signHash(message), key.PrivateKey) + signature, err := crypto.Sign(accounts.TextHash(message), key.PrivateKey) if err != nil { utils.Fatalf("Failed to sign message: %v", err) } @@ -88,7 +89,7 @@ type outputVerify struct { RecoveredPublicKey string } -var commandVerifyMessage = cli.Command{ +var commandVerifyMessage = &cli.Command{ Name: "verifymessage", Usage: "verify the signature of a signed message", ArgsUsage: "
", @@ -113,7 +114,7 @@ It is possible to refer to a file containing the message.`, utils.Fatalf("Signature encoding is not hexadecimal: %v", err) } - recoveredPubkey, err := crypto.SigToPub(signHash(message), signature) + recoveredPubkey, err := crypto.SigToPub(accounts.TextHash(message), signature) if err != nil || recoveredPubkey == nil { utils.Fatalf("Signature verification failed: %v", err) } @@ -143,7 +144,7 @@ It is possible to refer to a file containing the message.`, func getMessage(ctx *cli.Context, msgarg int) []byte { if file := ctx.String(msgfileFlag.Name); file != "" { - if len(ctx.Args()) > msgarg { + if ctx.NArg() > msgarg { utils.Fatalf("Can't use --msgfile and message argument at the same time.") } msg, err := os.ReadFile(file) @@ -151,9 +152,9 @@ func getMessage(ctx *cli.Context, msgarg int) []byte { utils.Fatalf("Can't read message file: %v", err) } return msg - } else if len(ctx.Args()) == msgarg+1 { + } else if ctx.NArg() == msgarg+1 { return []byte(ctx.Args().Get(msgarg)) } - utils.Fatalf("Invalid number of arguments: want %d, got %d", msgarg+1, len(ctx.Args())) + utils.Fatalf("Invalid number of arguments: want %d, got %d", msgarg+1, ctx.NArg()) return nil } diff --git a/cmd/ethkey/utils.go b/cmd/ethkey/utils.go index b81e70913b5b4..2821145089ec6 100644 --- a/cmd/ethkey/utils.go +++ b/cmd/ethkey/utils.go @@ -23,8 +23,7 @@ import ( "strings" "github.com/ethereum/go-ethereum/cmd/utils" - "github.com/ethereum/go-ethereum/crypto" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) // getPassphrase obtains a passphrase given by the user. It first checks the @@ -46,18 +45,6 @@ func getPassphrase(ctx *cli.Context, confirmation bool) string { return utils.GetPassPhrase("", confirmation) } -// signHash is a helper function that calculates a hash for the given message -// that can be safely used to calculate a signature from. -// -// The hash is calculated as -// keccak256("\x19Ethereum Signed Message:\n"${message length}${message}). -// -// This gives context to the signed message and prevents signing of transactions. -func signHash(data []byte) []byte { - msg := fmt.Sprintf("\x19Ethereum Signed Message:\n%d%s", len(data), data) - return crypto.Keccak256([]byte(msg)) -} - // mustPrintJSON prints the JSON encoding of the given object and // exits the program with an error message when the marshaling fails. func mustPrintJSON(jsonObject interface{}) { diff --git a/cmd/evm/compiler.go b/cmd/evm/compiler.go index 880f995f057cd..699d434bb0e13 100644 --- a/cmd/evm/compiler.go +++ b/cmd/evm/compiler.go @@ -23,10 +23,10 @@ import ( "github.com/ethereum/go-ethereum/cmd/evm/internal/compiler" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) -var compileCommand = cli.Command{ +var compileCommand = &cli.Command{ Action: compileCmd, Name: "compile", Usage: "compiles easm source to evm binary", @@ -34,7 +34,7 @@ var compileCommand = cli.Command{ } func compileCmd(ctx *cli.Context) error { - debug := ctx.GlobalBool(DebugFlag.Name) + debug := ctx.Bool(DebugFlag.Name) if len(ctx.Args().First()) == 0 { return errors.New("filename required") diff --git a/cmd/evm/disasm.go b/cmd/evm/disasm.go index 918b01376791a..a6a16fd13b77c 100644 --- a/cmd/evm/disasm.go +++ b/cmd/evm/disasm.go @@ -23,10 +23,10 @@ import ( "strings" "github.com/ethereum/go-ethereum/core/asm" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) -var disasmCommand = cli.Command{ +var disasmCommand = &cli.Command{ Action: disasmCmd, Name: "disasm", Usage: "disassembles evm binary", @@ -43,8 +43,8 @@ func disasmCmd(ctx *cli.Context) error { return err } in = string(input) - case ctx.GlobalIsSet(InputFlag.Name): - in = ctx.GlobalString(InputFlag.Name) + case ctx.IsSet(InputFlag.Name): + in = ctx.String(InputFlag.Name) default: return errors.New("missing filename or --input value") } diff --git a/cmd/evm/internal/t8ntool/block.go b/cmd/evm/internal/t8ntool/block.go index 9839afd5f488b..4a070b6c71b54 100644 --- a/cmd/evm/internal/t8ntool/block.go +++ b/cmd/evm/internal/t8ntool/block.go @@ -33,7 +33,7 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) //go:generate go run github.com/fjl/gencodec -type header -field-override headerMarshaling -out gen_header.go diff --git a/cmd/evm/internal/t8ntool/execution.go b/cmd/evm/internal/t8ntool/execution.go index 83a0025344a48..77f6ec37158be 100644 --- a/cmd/evm/internal/t8ntool/execution.go +++ b/cmd/evm/internal/t8ntool/execution.go @@ -100,7 +100,6 @@ type rejectedTx struct { func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, txs types.Transactions, miningReward int64, getTracerFn func(txIndex int, txHash common.Hash) (tracer vm.EVMLogger, err error)) (*state.StateDB, *ExecutionResult, error) { - // Capture errors for BLOCKHASH operation, if we haven't been supplied the // required blockhashes var hashError error @@ -241,7 +240,7 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, minerReward.Add(minerReward, perOmmer) // Add (8-delta)/8 reward := big.NewInt(8) - reward.Sub(reward, big.NewInt(0).SetUint64(ommer.Delta)) + reward.Sub(reward, new(big.Int).SetUint64(ommer.Delta)) reward.Mul(reward, blockReward) reward.Div(reward, big.NewInt(8)) statedb.AddBalance(ommer.Address, reward) @@ -269,7 +268,7 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, } func MakePreState(db ethdb.Database, accounts core.GenesisAlloc) *state.StateDB { - sdb := state.NewDatabase(db) + sdb := state.NewDatabaseWithConfig(db, &trie.Config{Preimages: true}) statedb, _ := state.New(common.Hash{}, sdb, nil) for addr, a := range accounts { statedb.SetCode(addr, a.Code) diff --git a/cmd/evm/internal/t8ntool/flags.go b/cmd/evm/internal/t8ntool/flags.go index de666f1151231..626220315e19e 100644 --- a/cmd/evm/internal/t8ntool/flags.go +++ b/cmd/evm/internal/t8ntool/flags.go @@ -22,45 +22,47 @@ import ( "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/tests" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) var ( - TraceFlag = cli.BoolFlag{ + TraceFlag = &cli.BoolFlag{ Name: "trace", Usage: "Output full trace logs to files .jsonl", } - TraceDisableMemoryFlag = cli.BoolTFlag{ + TraceDisableMemoryFlag = &cli.BoolFlag{ Name: "trace.nomemory", + Value: true, Usage: "Disable full memory dump in traces (deprecated)", } - TraceEnableMemoryFlag = cli.BoolFlag{ + TraceEnableMemoryFlag = &cli.BoolFlag{ Name: "trace.memory", Usage: "Enable full memory dump in traces", } - TraceDisableStackFlag = cli.BoolFlag{ + TraceDisableStackFlag = &cli.BoolFlag{ Name: "trace.nostack", Usage: "Disable stack output in traces", } - TraceDisableReturnDataFlag = cli.BoolTFlag{ + TraceDisableReturnDataFlag = &cli.BoolFlag{ Name: "trace.noreturndata", + Value: true, Usage: "Disable return data output in traces (deprecated)", } - TraceEnableReturnDataFlag = cli.BoolFlag{ + TraceEnableReturnDataFlag = &cli.BoolFlag{ Name: "trace.returndata", Usage: "Enable return data output in traces", } - OutputBasedir = cli.StringFlag{ + OutputBasedir = &cli.StringFlag{ Name: "output.basedir", Usage: "Specifies where output files are placed. Will be created if it does not exist.", Value: "", } - OutputBodyFlag = cli.StringFlag{ + OutputBodyFlag = &cli.StringFlag{ Name: "output.body", Usage: "If set, the RLP of the transactions (block body) will be written to this file.", Value: "", } - OutputAllocFlag = cli.StringFlag{ + OutputAllocFlag = &cli.StringFlag{ Name: "output.alloc", Usage: "Determines where to put the `alloc` of the post-state.\n" + "\t`stdout` - into the stdout output\n" + @@ -68,7 +70,7 @@ var ( "\t - into the file ", Value: "alloc.json", } - OutputResultFlag = cli.StringFlag{ + OutputResultFlag = &cli.StringFlag{ Name: "output.result", Usage: "Determines where to put the `result` (stateroot, txroot etc) of the post-state.\n" + "\t`stdout` - into the stdout output\n" + @@ -76,7 +78,7 @@ var ( "\t - into the file ", Value: "result.json", } - OutputBlockFlag = cli.StringFlag{ + OutputBlockFlag = &cli.StringFlag{ Name: "output.block", Usage: "Determines where to put the `block` after building.\n" + "\t`stdout` - into the stdout output\n" + @@ -84,65 +86,65 @@ var ( "\t - into the file ", Value: "block.json", } - InputAllocFlag = cli.StringFlag{ + InputAllocFlag = &cli.StringFlag{ Name: "input.alloc", Usage: "`stdin` or file name of where to find the prestate alloc to use.", Value: "alloc.json", } - InputEnvFlag = cli.StringFlag{ + InputEnvFlag = &cli.StringFlag{ Name: "input.env", Usage: "`stdin` or file name of where to find the prestate env to use.", Value: "env.json", } - InputTxsFlag = cli.StringFlag{ + InputTxsFlag = &cli.StringFlag{ Name: "input.txs", Usage: "`stdin` or file name of where to find the transactions to apply. " + "If the file extension is '.rlp', then the data is interpreted as an RLP list of signed transactions." + "The '.rlp' format is identical to the output.body format.", Value: "txs.json", } - InputHeaderFlag = cli.StringFlag{ + InputHeaderFlag = &cli.StringFlag{ Name: "input.header", Usage: "`stdin` or file name of where to find the block header to use.", Value: "header.json", } - InputOmmersFlag = cli.StringFlag{ + InputOmmersFlag = &cli.StringFlag{ Name: "input.ommers", Usage: "`stdin` or file name of where to find the list of ommer header RLPs to use.", } - InputTxsRlpFlag = cli.StringFlag{ + InputTxsRlpFlag = &cli.StringFlag{ Name: "input.txs", Usage: "`stdin` or file name of where to find the transactions list in RLP form.", Value: "txs.rlp", } - SealCliqueFlag = cli.StringFlag{ + SealCliqueFlag = &cli.StringFlag{ Name: "seal.clique", Usage: "Seal block with Clique. `stdin` or file name of where to find the Clique sealing data.", } - SealEthashFlag = cli.BoolFlag{ + SealEthashFlag = &cli.BoolFlag{ Name: "seal.ethash", Usage: "Seal block with ethash.", } - SealEthashDirFlag = cli.StringFlag{ + SealEthashDirFlag = &cli.StringFlag{ Name: "seal.ethash.dir", Usage: "Path to ethash DAG. If none exists, a new DAG will be generated.", } - SealEthashModeFlag = cli.StringFlag{ + SealEthashModeFlag = &cli.StringFlag{ Name: "seal.ethash.mode", Usage: "Defines the type and amount of PoW verification an ethash engine makes.", Value: "normal", } - RewardFlag = cli.Int64Flag{ + RewardFlag = &cli.Int64Flag{ Name: "state.reward", Usage: "Mining reward. Set to -1 to disable", Value: 0, } - ChainIDFlag = cli.Int64Flag{ + ChainIDFlag = &cli.Int64Flag{ Name: "state.chainid", Usage: "ChainID to use", Value: 1, } - ForknameFlag = cli.StringFlag{ + ForknameFlag = &cli.StringFlag{ Name: "state.fork", Usage: fmt.Sprintf("Name of ruleset to use."+ "\n\tAvailable forknames:"+ @@ -152,9 +154,9 @@ var ( "\n\tSyntax (+ExtraEip)", strings.Join(tests.AvailableForks(), "\n\t "), strings.Join(vm.ActivateableEips(), ", ")), - Value: "ArrowGlacier", + Value: "GrayGlacier", } - VerbosityFlag = cli.IntFlag{ + VerbosityFlag = &cli.IntFlag{ Name: "verbosity", Usage: "sets the verbosity level", Value: 3, diff --git a/cmd/evm/internal/t8ntool/transaction.go b/cmd/evm/internal/t8ntool/transaction.go index 6f1c964ada026..3409c0a3bf01f 100644 --- a/cmd/evm/internal/t8ntool/transaction.go +++ b/cmd/evm/internal/t8ntool/transaction.go @@ -32,7 +32,7 @@ import ( "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/tests" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) type result struct { diff --git a/cmd/evm/internal/t8ntool/transition.go b/cmd/evm/internal/t8ntool/transition.go index b254baa995827..e2d9cced22551 100644 --- a/cmd/evm/internal/t8ntool/transition.go +++ b/cmd/evm/internal/t8ntool/transition.go @@ -38,7 +38,7 @@ import ( "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/tests" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) const ( @@ -334,8 +334,9 @@ func (t *txWithKey) UnmarshalJSON(input []byte) error { // signUnsignedTransactions converts the input txs to canonical transactions. // // The transactions can have two forms, either -// 1. unsigned or -// 2. signed +// 1. unsigned or +// 2. signed +// // For (1), r, s, v, need so be zero, and the `secretKey` needs to be set. // If so, we sign it here and now, with the given `secretKey` // If the condition above is not met, then it's considered a signed transaction. diff --git a/cmd/evm/internal/t8ntool/utils.go b/cmd/evm/internal/t8ntool/utils.go index 1c54f09bf4176..8ec38c7618de0 100644 --- a/cmd/evm/internal/t8ntool/utils.go +++ b/cmd/evm/internal/t8ntool/utils.go @@ -21,7 +21,7 @@ import ( "fmt" "os" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) // readFile reads the json-data in the provided path and marshals into dest. diff --git a/cmd/evm/main.go b/cmd/evm/main.go index 2f404d48e9031..5f9e75f48c6f0 100644 --- a/cmd/evm/main.go +++ b/cmd/evm/main.go @@ -23,115 +23,111 @@ import ( "os" "github.com/ethereum/go-ethereum/cmd/evm/internal/t8ntool" - "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/internal/flags" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) -var gitCommit = "" // Git SHA1 commit hash of the release (set via linker flags) -var gitDate = "" - var ( - app = flags.NewApp(gitCommit, gitDate, "the evm command line interface") - - DebugFlag = cli.BoolFlag{ + DebugFlag = &cli.BoolFlag{ Name: "debug", Usage: "output full trace logs", } - MemProfileFlag = cli.StringFlag{ + MemProfileFlag = &cli.StringFlag{ Name: "memprofile", Usage: "creates a memory profile at the given path", } - CPUProfileFlag = cli.StringFlag{ + CPUProfileFlag = &cli.StringFlag{ Name: "cpuprofile", Usage: "creates a CPU profile at the given path", } - StatDumpFlag = cli.BoolFlag{ + StatDumpFlag = &cli.BoolFlag{ Name: "statdump", Usage: "displays stack and heap memory information", } - CodeFlag = cli.StringFlag{ + CodeFlag = &cli.StringFlag{ Name: "code", Usage: "EVM code", } - CodeFileFlag = cli.StringFlag{ + CodeFileFlag = &cli.StringFlag{ Name: "codefile", Usage: "File containing EVM code. If '-' is specified, code is read from stdin ", } - GasFlag = cli.Uint64Flag{ + GasFlag = &cli.Uint64Flag{ Name: "gas", Usage: "gas limit for the evm", Value: 10000000000, } - PriceFlag = utils.BigFlag{ + PriceFlag = &flags.BigFlag{ Name: "price", Usage: "price set for the evm", Value: new(big.Int), } - ValueFlag = utils.BigFlag{ + ValueFlag = &flags.BigFlag{ Name: "value", Usage: "value set for the evm", Value: new(big.Int), } - DumpFlag = cli.BoolFlag{ + DumpFlag = &cli.BoolFlag{ Name: "dump", Usage: "dumps the state after the run", } - InputFlag = cli.StringFlag{ + InputFlag = &cli.StringFlag{ Name: "input", Usage: "input for the EVM", } - InputFileFlag = cli.StringFlag{ + InputFileFlag = &cli.StringFlag{ Name: "inputfile", Usage: "file containing input for the EVM", } - VerbosityFlag = cli.IntFlag{ + VerbosityFlag = &cli.IntFlag{ Name: "verbosity", Usage: "sets the verbosity level", } - BenchFlag = cli.BoolFlag{ + BenchFlag = &cli.BoolFlag{ Name: "bench", Usage: "benchmark the execution", } - CreateFlag = cli.BoolFlag{ + CreateFlag = &cli.BoolFlag{ Name: "create", Usage: "indicates the action should be create rather than call", } - GenesisFlag = cli.StringFlag{ + GenesisFlag = &cli.StringFlag{ Name: "prestate", Usage: "JSON file with prestate (genesis) config", } - MachineFlag = cli.BoolFlag{ + MachineFlag = &cli.BoolFlag{ Name: "json", Usage: "output trace logs in machine readable format (json)", } - SenderFlag = cli.StringFlag{ + SenderFlag = &cli.StringFlag{ Name: "sender", Usage: "The transaction origin", } - ReceiverFlag = cli.StringFlag{ + ReceiverFlag = &cli.StringFlag{ Name: "receiver", Usage: "The transaction receiver (execution context)", } - DisableMemoryFlag = cli.BoolTFlag{ + DisableMemoryFlag = &cli.BoolFlag{ Name: "nomemory", + Value: true, Usage: "disable memory output", } - DisableStackFlag = cli.BoolFlag{ + DisableStackFlag = &cli.BoolFlag{ Name: "nostack", Usage: "disable stack output", } - DisableStorageFlag = cli.BoolFlag{ + DisableStorageFlag = &cli.BoolFlag{ Name: "nostorage", Usage: "disable storage output", } - DisableReturnDataFlag = cli.BoolTFlag{ + DisableReturnDataFlag = &cli.BoolFlag{ Name: "noreturndata", + Value: true, Usage: "enable return data output", } ) -var stateTransitionCommand = cli.Command{ +var stateTransitionCommand = &cli.Command{ Name: "transition", Aliases: []string{"t8n"}, Usage: "executes a full state transition", @@ -156,7 +152,8 @@ var stateTransitionCommand = cli.Command{ t8ntool.VerbosityFlag, }, } -var transactionCommand = cli.Command{ + +var transactionCommand = &cli.Command{ Name: "transaction", Aliases: []string{"t9n"}, Usage: "performs transaction validation", @@ -169,7 +166,7 @@ var transactionCommand = cli.Command{ }, } -var blockBuilderCommand = cli.Command{ +var blockBuilderCommand = &cli.Command{ Name: "block-builder", Aliases: []string{"b11r"}, Usage: "builds a block", @@ -188,6 +185,8 @@ var blockBuilderCommand = cli.Command{ }, } +var app = flags.NewApp("the evm command line interface") + func init() { app.Flags = []cli.Flag{ BenchFlag, @@ -214,7 +213,7 @@ func init() { DisableStorageFlag, DisableReturnDataFlag, } - app.Commands = []cli.Command{ + app.Commands = []*cli.Command{ compileCommand, disasmCommand, runCommand, @@ -223,7 +222,6 @@ func init() { transactionCommand, blockBuilderCommand, } - cli.CommandHelpTemplate = flags.OriginCommandHelpTemplate } func main() { diff --git a/cmd/evm/runner.go b/cmd/evm/runner.go index 5680c07a40ee4..9b1975c0500ef 100644 --- a/cmd/evm/runner.go +++ b/cmd/evm/runner.go @@ -37,12 +37,13 @@ import ( "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/core/vm/runtime" "github.com/ethereum/go-ethereum/eth/tracers/logger" + "github.com/ethereum/go-ethereum/internal/flags" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) -var runCommand = cli.Command{ +var runCommand = &cli.Command{ Action: runCmd, Name: "run", Usage: "run arbitrary evm binary", @@ -106,14 +107,14 @@ func timedExec(bench bool, execFunc func() ([]byte, uint64, error)) (output []by func runCmd(ctx *cli.Context) error { glogger := log.NewGlogHandler(log.StreamHandler(os.Stderr, log.TerminalFormat(false))) - glogger.Verbosity(log.Lvl(ctx.GlobalInt(VerbosityFlag.Name))) + glogger.Verbosity(log.Lvl(ctx.Int(VerbosityFlag.Name))) log.Root().SetHandler(glogger) logconfig := &logger.Config{ - EnableMemory: !ctx.GlobalBool(DisableMemoryFlag.Name), - DisableStack: ctx.GlobalBool(DisableStackFlag.Name), - DisableStorage: ctx.GlobalBool(DisableStorageFlag.Name), - EnableReturnData: !ctx.GlobalBool(DisableReturnDataFlag.Name), - Debug: ctx.GlobalBool(DebugFlag.Name), + EnableMemory: !ctx.Bool(DisableMemoryFlag.Name), + DisableStack: ctx.Bool(DisableStackFlag.Name), + DisableStorage: ctx.Bool(DisableStorageFlag.Name), + EnableReturnData: !ctx.Bool(DisableReturnDataFlag.Name), + Debug: ctx.Bool(DebugFlag.Name), } var ( @@ -125,37 +126,37 @@ func runCmd(ctx *cli.Context) error { receiver = common.BytesToAddress([]byte("receiver")) genesisConfig *core.Genesis ) - if ctx.GlobalBool(MachineFlag.Name) { + if ctx.Bool(MachineFlag.Name) { tracer = logger.NewJSONLogger(logconfig, os.Stdout) - } else if ctx.GlobalBool(DebugFlag.Name) { + } else if ctx.Bool(DebugFlag.Name) { debugLogger = logger.NewStructLogger(logconfig) tracer = debugLogger } else { debugLogger = logger.NewStructLogger(logconfig) } - if ctx.GlobalString(GenesisFlag.Name) != "" { - gen := readGenesis(ctx.GlobalString(GenesisFlag.Name)) + if ctx.String(GenesisFlag.Name) != "" { + gen := readGenesis(ctx.String(GenesisFlag.Name)) genesisConfig = gen db := rawdb.NewMemoryDatabase() - genesis := gen.ToBlock(db) + genesis := gen.MustCommit(db) statedb, _ = state.New(genesis.Root(), state.NewDatabase(db), nil) chainConfig = gen.Config } else { statedb, _ = state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) genesisConfig = new(core.Genesis) } - if ctx.GlobalString(SenderFlag.Name) != "" { - sender = common.HexToAddress(ctx.GlobalString(SenderFlag.Name)) + if ctx.String(SenderFlag.Name) != "" { + sender = common.HexToAddress(ctx.String(SenderFlag.Name)) } statedb.CreateAccount(sender) - if ctx.GlobalString(ReceiverFlag.Name) != "" { - receiver = common.HexToAddress(ctx.GlobalString(ReceiverFlag.Name)) + if ctx.String(ReceiverFlag.Name) != "" { + receiver = common.HexToAddress(ctx.String(ReceiverFlag.Name)) } var code []byte - codeFileFlag := ctx.GlobalString(CodeFileFlag.Name) - codeFlag := ctx.GlobalString(CodeFlag.Name) + codeFileFlag := ctx.String(CodeFileFlag.Name) + codeFlag := ctx.String(CodeFlag.Name) // The '--code' or '--codefile' flag overrides code in state if codeFileFlag != "" || codeFlag != "" { @@ -197,7 +198,7 @@ func runCmd(ctx *cli.Context) error { } code = common.Hex2Bytes(bin) } - initialGas := ctx.GlobalUint64(GasFlag.Name) + initialGas := ctx.Uint64(GasFlag.Name) if genesisConfig.GasLimit != 0 { initialGas = genesisConfig.GasLimit } @@ -205,19 +206,19 @@ func runCmd(ctx *cli.Context) error { Origin: sender, State: statedb, GasLimit: initialGas, - GasPrice: utils.GlobalBig(ctx, PriceFlag.Name), - Value: utils.GlobalBig(ctx, ValueFlag.Name), + GasPrice: flags.GlobalBig(ctx, PriceFlag.Name), + Value: flags.GlobalBig(ctx, ValueFlag.Name), Difficulty: genesisConfig.Difficulty, Time: new(big.Int).SetUint64(genesisConfig.Timestamp), Coinbase: genesisConfig.Coinbase, BlockNumber: new(big.Int).SetUint64(genesisConfig.Number), EVMConfig: vm.Config{ Tracer: tracer, - Debug: ctx.GlobalBool(DebugFlag.Name) || ctx.GlobalBool(MachineFlag.Name), + Debug: ctx.Bool(DebugFlag.Name) || ctx.Bool(MachineFlag.Name), }, } - if cpuProfilePath := ctx.GlobalString(CPUProfileFlag.Name); cpuProfilePath != "" { + if cpuProfilePath := ctx.String(CPUProfileFlag.Name); cpuProfilePath != "" { f, err := os.Create(cpuProfilePath) if err != nil { fmt.Println("could not create CPU profile: ", err) @@ -237,14 +238,14 @@ func runCmd(ctx *cli.Context) error { } var hexInput []byte - if inputFileFlag := ctx.GlobalString(InputFileFlag.Name); inputFileFlag != "" { + if inputFileFlag := ctx.String(InputFileFlag.Name); inputFileFlag != "" { var err error if hexInput, err = os.ReadFile(inputFileFlag); err != nil { fmt.Printf("could not load input from file: %v\n", err) os.Exit(1) } } else { - hexInput = []byte(ctx.GlobalString(InputFlag.Name)) + hexInput = []byte(ctx.String(InputFlag.Name)) } hexInput = bytes.TrimSpace(hexInput) if len(hexInput)%2 != 0 { @@ -254,7 +255,7 @@ func runCmd(ctx *cli.Context) error { input := common.FromHex(string(hexInput)) var execFunc func() ([]byte, uint64, error) - if ctx.GlobalBool(CreateFlag.Name) { + if ctx.Bool(CreateFlag.Name) { input = append(code, input...) execFunc = func() ([]byte, uint64, error) { output, _, gasLeft, err := runtime.Create(input, &runtimeConfig) @@ -269,16 +270,16 @@ func runCmd(ctx *cli.Context) error { } } - bench := ctx.GlobalBool(BenchFlag.Name) + bench := ctx.Bool(BenchFlag.Name) output, leftOverGas, stats, err := timedExec(bench, execFunc) - if ctx.GlobalBool(DumpFlag.Name) { + if ctx.Bool(DumpFlag.Name) { statedb.Commit(true) statedb.IntermediateRoot(true) fmt.Println(string(statedb.Dump(nil))) } - if memProfilePath := ctx.GlobalString(MemProfileFlag.Name); memProfilePath != "" { + if memProfilePath := ctx.String(MemProfileFlag.Name); memProfilePath != "" { f, err := os.Create(memProfilePath) if err != nil { fmt.Println("could not create memory profile: ", err) @@ -291,7 +292,7 @@ func runCmd(ctx *cli.Context) error { f.Close() } - if ctx.GlobalBool(DebugFlag.Name) { + if ctx.Bool(DebugFlag.Name) { if debugLogger != nil { fmt.Fprintln(os.Stderr, "#### TRACE ####") logger.WriteTrace(os.Stderr, debugLogger.StructLogs()) @@ -300,7 +301,7 @@ func runCmd(ctx *cli.Context) error { logger.WriteLogs(os.Stderr, statedb.Logs()) } - if bench || ctx.GlobalBool(StatDumpFlag.Name) { + if bench || ctx.Bool(StatDumpFlag.Name) { fmt.Fprintf(os.Stderr, `EVM gas used: %d execution time: %v allocations: %d @@ -308,7 +309,7 @@ allocated bytes: %d `, initialGas-leftOverGas, stats.time, stats.allocs, stats.bytesAllocated) } if tracer == nil { - fmt.Printf("0x%x\n", output) + fmt.Printf("%#x\n", output) if err != nil { fmt.Printf(" error: %v\n", err) } diff --git a/cmd/evm/staterunner.go b/cmd/evm/staterunner.go index fcdac33eedfb9..36f4e19b0bea6 100644 --- a/cmd/evm/staterunner.go +++ b/cmd/evm/staterunner.go @@ -28,10 +28,10 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/tests" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) -var stateTestCommand = cli.Command{ +var stateTestCommand = &cli.Command{ Action: stateTestCmd, Name: "statetest", Usage: "executes the given state tests", @@ -54,25 +54,25 @@ func stateTestCmd(ctx *cli.Context) error { } // Configure the go-ethereum logger glogger := log.NewGlogHandler(log.StreamHandler(os.Stderr, log.TerminalFormat(false))) - glogger.Verbosity(log.Lvl(ctx.GlobalInt(VerbosityFlag.Name))) + glogger.Verbosity(log.Lvl(ctx.Int(VerbosityFlag.Name))) log.Root().SetHandler(glogger) // Configure the EVM logger config := &logger.Config{ - EnableMemory: !ctx.GlobalBool(DisableMemoryFlag.Name), - DisableStack: ctx.GlobalBool(DisableStackFlag.Name), - DisableStorage: ctx.GlobalBool(DisableStorageFlag.Name), - EnableReturnData: !ctx.GlobalBool(DisableReturnDataFlag.Name), + EnableMemory: !ctx.Bool(DisableMemoryFlag.Name), + DisableStack: ctx.Bool(DisableStackFlag.Name), + DisableStorage: ctx.Bool(DisableStorageFlag.Name), + EnableReturnData: !ctx.Bool(DisableReturnDataFlag.Name), } var ( tracer vm.EVMLogger debugger *logger.StructLogger ) switch { - case ctx.GlobalBool(MachineFlag.Name): + case ctx.Bool(MachineFlag.Name): tracer = logger.NewJSONLogger(config, os.Stderr) - case ctx.GlobalBool(DebugFlag.Name): + case ctx.Bool(DebugFlag.Name): debugger = logger.NewStructLogger(config) tracer = debugger @@ -91,7 +91,7 @@ func stateTestCmd(ctx *cli.Context) error { // Iterate over all the tests, run them and aggregate the results cfg := vm.Config{ Tracer: tracer, - Debug: ctx.GlobalBool(DebugFlag.Name) || ctx.GlobalBool(MachineFlag.Name), + Debug: ctx.Bool(DebugFlag.Name) || ctx.Bool(MachineFlag.Name), } results := make([]StatetestResult, 0, len(tests)) for key, test := range tests { @@ -100,13 +100,13 @@ func stateTestCmd(ctx *cli.Context) error { result := &StatetestResult{Name: key, Fork: st.Fork, Pass: true} _, s, err := test.Run(st, cfg, false) // print state root for evmlab tracing - if ctx.GlobalBool(MachineFlag.Name) && s != nil { + if ctx.Bool(MachineFlag.Name) && s != nil { fmt.Fprintf(os.Stderr, "{\"stateRoot\": \"%x\"}\n", s.IntermediateRoot(false)) } if err != nil { // Test failed, mark as so and dump any state to aid debugging result.Pass, result.Error = false, err.Error() - if ctx.GlobalBool(DumpFlag.Name) && s != nil { + if ctx.Bool(DumpFlag.Name) && s != nil { dump := s.RawDump(nil) result.State = &dump } @@ -115,7 +115,7 @@ func stateTestCmd(ctx *cli.Context) error { results = append(results, *result) // Print any structured logs collected - if ctx.GlobalBool(DebugFlag.Name) { + if ctx.Bool(DebugFlag.Name) { if debugger != nil { fmt.Fprintln(os.Stderr, "#### TRACE ####") logger.WriteTrace(os.Stderr, debugger.StructLogs()) diff --git a/cmd/evm/t8n_test.go b/cmd/evm/t8n_test.go index 92c01398ba364..72c062e8d9236 100644 --- a/cmd/evm/t8n_test.go +++ b/cmd/evm/t8n_test.go @@ -211,6 +211,14 @@ func TestT8n(t *testing.T) { output: t8nOutput{result: true}, expOut: "exp_arrowglacier.json", }, + { // Difficulty calculation on gray glacier + base: "./testdata/19", + input: t8nInput{ + "alloc.json", "txs.json", "env.json", "GrayGlacier", "", + }, + output: t8nOutput{result: true}, + expOut: "exp_grayglacier.json", + }, { // Sign unprotected (pre-EIP155) transaction base: "./testdata/23", input: t8nInput{ @@ -236,7 +244,6 @@ func TestT8n(t *testing.T) { expExitCode: 3, }, } { - args := []string{"t8n"} args = append(args, tc.output.get()...) args = append(args, tc.input.get(tc.base)...) @@ -347,7 +354,6 @@ func TestT9n(t *testing.T) { expExitCode: t8ntool.ErrorIO, }, } { - args := []string{"t9n"} args = append(args, tc.input.get(tc.base)...) @@ -467,7 +473,6 @@ func TestB11r(t *testing.T) { expOut: "exp.json", }, } { - args := []string{"b11r"} args = append(args, tc.input.get(tc.base)...) diff --git a/cmd/evm/testdata/19/exp_grayglacier.json b/cmd/evm/testdata/19/exp_grayglacier.json new file mode 100644 index 0000000000000..95a3cb1685cfe --- /dev/null +++ b/cmd/evm/testdata/19/exp_grayglacier.json @@ -0,0 +1,12 @@ +{ + "result": { + "stateRoot": "0x6f058887ca01549716789c380ede95aecc510e6d1fdc4dbf67d053c7c07f4bdc", + "txRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "receiptsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "logsHash": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "receipts": [], + "currentDifficulty": "0x2000000004000", + "gasUsed": "0x0" + } +} \ No newline at end of file diff --git a/cmd/evm/testdata/19/readme.md b/cmd/evm/testdata/19/readme.md index 5fae183f48860..095d4525d4fe9 100644 --- a/cmd/evm/testdata/19/readme.md +++ b/cmd/evm/testdata/19/readme.md @@ -1,9 +1,9 @@ ## Difficulty calculation This test shows how the `evm t8n` can be used to calculate the (ethash) difficulty, if none is provided by the caller, -this time on `ArrowGlacier` (Eip 4345). +this time on `GrayGlacier` (Eip 5133). -Calculating it (with an empty set of txs) using `ArrowGlacier` rules (and no provided unclehash for the parent block): +Calculating it (with an empty set of txs) using `GrayGlacier` rules (and no provided unclehash for the parent block): ``` -[user@work evm]$ ./evm t8n --input.alloc=./testdata/14/alloc.json --input.txs=./testdata/14/txs.json --input.env=./testdata/14/env.json --output.result=stdout --state.fork=ArrowGlacier +[user@work evm]$ ./evm t8n --input.alloc=./testdata/19/alloc.json --input.txs=./testdata/19/txs.json --input.env=./testdata/19/env.json --output.result=stdout --state.fork=GrayGlacier ``` \ No newline at end of file diff --git a/cmd/faucet/README.md b/cmd/faucet/README.md index f27e94aa9e72c..6e9b14ebc78ae 100644 --- a/cmd/faucet/README.md +++ b/cmd/faucet/README.md @@ -2,17 +2,18 @@ The `faucet` is a simplistic web application with the goal of distributing small amounts of Ether in private and test networks. -Users need to post their Ethereum addresses to fund in a Twitter status update or public Facebook post and share the link to the faucet. The faucet will in turn deduplicate user requests and send the Ether. After a funding round, the faucet prevents the same user requesting again for a pre-configured amount of time, proportional to the amount of Ether requested. +Users need to post their Ethereum addresses to fund in a Twitter status update or public Facebook post and share the link to the faucet. The faucet will in turn deduplicate user requests and send the Ether. After a funding round, the faucet prevents the same user from requesting again for a pre-configured amount of time, proportional to the amount of Ether requested. ## Operation The `faucet` is a single binary app (everything included) with all configurations set via command line flags and a few files. -First thing's first, the `faucet` needs to connect to an Ethereum network, for which it needs the necessary genesis and network infos. Each of the following flags must be set: +First things first, the `faucet` needs to connect to an Ethereum network, for which it needs the necessary genesis and network infos. Each of the following flags must be set: -- `-genesis` is a path to a file containin the network `genesis.json`. or using: +- `-genesis` is a path to a file containing the network `genesis.json`. or using: - `-goerli` with the faucet with Görli network config - `-rinkeby` with the faucet with Rinkeby network config + - `-sepolia` with the faucet with Sepolia network config - `-network` is the devp2p network id used during connection - `-bootnodes` is a list of `enode://` ids to join the network through @@ -49,4 +50,4 @@ Sybil protection via Facebook uses the website to directly download post data th ## Miscellaneous -Beside the above - mostly essential - CLI flags, there are a number that can be used to fine tune the `faucet`'s operation. Please see `faucet --help` for a full list. \ No newline at end of file +Beside the above - mostly essential - CLI flags, there are a number that can be used to fine-tune the `faucet`'s operation. Please see `faucet --help` for a full list. diff --git a/cmd/faucet/faucet.go b/cmd/faucet/faucet.go index bcb837062f6fd..bec1f6d33b8e8 100644 --- a/cmd/faucet/faucet.go +++ b/cmd/faucet/faucet.go @@ -49,6 +49,7 @@ import ( "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/ethstats" + "github.com/ethereum/go-ethereum/internal/version" "github.com/ethereum/go-ethereum/les" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" @@ -86,17 +87,13 @@ var ( goerliFlag = flag.Bool("goerli", false, "Initializes the faucet with Görli network config") rinkebyFlag = flag.Bool("rinkeby", false, "Initializes the faucet with Rinkeby network config") + sepoliaFlag = flag.Bool("sepolia", false, "Initializes the faucet with Sepolia network config") ) var ( ether = new(big.Int).Exp(big.NewInt(10), big.NewInt(18), nil) ) -var ( - gitCommit = "" // Git SHA1 commit hash of the release (set via linker flags) - gitDate = "" // Git commit date YYYYMMDD of the release (set via linker flags) -) - //go:embed faucet.html var websiteTmpl string @@ -143,7 +140,7 @@ func main() { log.Crit("Failed to render the faucet template", "err", err) } // Load and parse the genesis block requested by the user - genesis, err := getGenesis(*genesisFlag, *goerliFlag, *rinkebyFlag) + genesis, err := getGenesis(*genesisFlag, *goerliFlag, *rinkebyFlag, *sepoliaFlag) if err != nil { log.Crit("Failed to parse genesis config", "err", err) } @@ -225,9 +222,10 @@ type wsConn struct { func newFaucet(genesis *core.Genesis, port int, enodes []*enode.Node, network uint64, stats string, ks *keystore.KeyStore, index []byte) (*faucet, error) { // Assemble the raw devp2p protocol stack + git, _ := version.VCS() stack, err := node.New(&node.Config{ Name: "geth", - Version: params.VersionWithCommit(gitCommit, gitDate), + Version: params.VersionWithCommit(git.Commit, git.Date), DataDir: filepath.Join(os.Getenv("HOME"), ".faucet"), P2P: p2p.Config{ NAT: nat.Any(), @@ -247,7 +245,7 @@ func newFaucet(genesis *core.Genesis, port int, enodes []*enode.Node, network ui cfg.SyncMode = downloader.LightSync cfg.NetworkId = network cfg.Genesis = genesis - utils.SetDNSDiscoveryDefaults(&cfg, genesis.ToBlock(nil).Hash()) + utils.SetDNSDiscoveryDefaults(&cfg, genesis.ToBlock().Hash()) lesBackend, err := les.New(stack, &cfg) if err != nil { @@ -708,7 +706,7 @@ func authTwitter(url string, tokenV1, tokenV2 string) (string, string, string, c case tokenV2 != "": return authTwitterWithTokenV2(tweetID, tokenV2) } - // Twiter API token isn't provided so we just load the public posts + // Twitter API token isn't provided so we just load the public posts // and scrape it for the Ethereum address and profile URL. We need to load // the mobile page though since the main page loads tweet contents via JS. url = strings.Replace(url, "https://twitter.com/", "https://mobile.twitter.com/", 1) @@ -860,7 +858,7 @@ func authFacebook(url string) (string, string, common.Address, error) { address := common.HexToAddress(string(regexp.MustCompile("0x[0-9a-fA-F]{40}").Find(body))) if address == (common.Address{}) { //lint:ignore ST1005 This error is to be displayed in the browser - return "", "", common.Address{}, errors.New("No Ethereum address found to fund") + return "", "", common.Address{}, errors.New("No Ethereum address found to fund. Please check the post URL and verify that it can be viewed publicly.") } var avatar string if parts = regexp.MustCompile(`src="([^"]+fbcdn\.net[^"]+)"`).FindStringSubmatch(string(body)); len(parts) == 2 { @@ -882,7 +880,7 @@ func authNoAuth(url string) (string, string, common.Address, error) { } // getGenesis returns a genesis based on input args -func getGenesis(genesisFlag string, goerliFlag bool, rinkebyFlag bool) (*core.Genesis, error) { +func getGenesis(genesisFlag string, goerliFlag bool, rinkebyFlag bool, sepoliaFlag bool) (*core.Genesis, error) { switch { case genesisFlag != "": var genesis core.Genesis @@ -892,6 +890,8 @@ func getGenesis(genesisFlag string, goerliFlag bool, rinkebyFlag bool) (*core.Ge return core.DefaultGoerliGenesisBlock(), nil case rinkebyFlag: return core.DefaultRinkebyGenesisBlock(), nil + case sepoliaFlag: + return core.DefaultSepoliaGenesisBlock(), nil default: return nil, fmt.Errorf("no genesis flag provided") } diff --git a/cmd/geth/accountcmd.go b/cmd/geth/accountcmd.go index 0b7d58e8888cd..5158b7606cdef 100644 --- a/cmd/geth/accountcmd.go +++ b/cmd/geth/accountcmd.go @@ -25,29 +25,27 @@ import ( "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) var ( - walletCommand = cli.Command{ + walletCommand = &cli.Command{ Name: "wallet", Usage: "Manage Ethereum presale wallets", ArgsUsage: "", - Category: "ACCOUNT COMMANDS", Description: ` geth wallet import /path/to/my/presale.wallet will prompt for your password and imports your ether presale account. It can be used non-interactively with the --password option taking a passwordfile as argument containing the wallet password in plaintext.`, - Subcommands: []cli.Command{ + Subcommands: []*cli.Command{ { Name: "import", Usage: "Import Ethereum presale wallet", ArgsUsage: "", - Action: utils.MigrateFlags(importWallet), - Category: "ACCOUNT COMMANDS", + Action: importWallet, Flags: []cli.Flag{ utils.DataDirFlag, utils.KeyStoreDirFlag, @@ -64,10 +62,9 @@ passwordfile as argument containing the wallet password in plaintext.`, }, } - accountCommand = cli.Command{ - Name: "account", - Usage: "Manage accounts", - Category: "ACCOUNT COMMANDS", + accountCommand = &cli.Command{ + Name: "account", + Usage: "Manage accounts", Description: ` Manage accounts, list all existing accounts, import a private key into a new @@ -88,11 +85,11 @@ It is safe to transfer the entire directory or the individual keys therein between ethereum nodes by simply copying. Make sure you backup your keys regularly.`, - Subcommands: []cli.Command{ + Subcommands: []*cli.Command{ { Name: "list", Usage: "Print summary of existing accounts", - Action: utils.MigrateFlags(accountList), + Action: accountList, Flags: []cli.Flag{ utils.DataDirFlag, utils.KeyStoreDirFlag, @@ -103,7 +100,7 @@ Print a short summary of all accounts`, { Name: "new", Usage: "Create a new account", - Action: utils.MigrateFlags(accountCreate), + Action: accountCreate, Flags: []cli.Flag{ utils.DataDirFlag, utils.KeyStoreDirFlag, @@ -128,7 +125,7 @@ password to file or expose in any other way. { Name: "update", Usage: "Update an existing account", - Action: utils.MigrateFlags(accountUpdate), + Action: accountUpdate, ArgsUsage: "
", Flags: []cli.Flag{ utils.DataDirFlag, @@ -157,7 +154,7 @@ changing your password is only possible interactively. { Name: "import", Usage: "Import a private key into a new account", - Action: utils.MigrateFlags(accountImport), + Action: accountImport, Flags: []cli.Flag{ utils.DataDirFlag, utils.KeyStoreDirFlag, @@ -239,14 +236,15 @@ func ambiguousAddrRecovery(ks *keystore.KeyStore, err *keystore.AmbiguousAddrErr } fmt.Println("Testing your password against all of them...") var match *accounts.Account - for _, a := range err.Matches { - if err := ks.Unlock(a, auth); err == nil { - match = &a + for i, a := range err.Matches { + if e := ks.Unlock(a, auth); e == nil { + match = &err.Matches[i] break } } if match == nil { utils.Fatalf("None of the listed files could be unlocked.") + return accounts.Account{} } fmt.Printf("Your password unlocked %s\n", match.URL) fmt.Println("In order to avoid this warning, you need to remove the following duplicate key files:") @@ -262,7 +260,7 @@ func ambiguousAddrRecovery(ks *keystore.KeyStore, err *keystore.AmbiguousAddrErr func accountCreate(ctx *cli.Context) error { cfg := gethConfig{Node: defaultNodeConfig()} // Load config file. - if file := ctx.GlobalString(configFileFlag.Name); file != "" { + if file := ctx.String(configFileFlag.Name); file != "" { if err := loadConfig(file, &cfg); err != nil { utils.Fatalf("%v", err) } @@ -299,13 +297,13 @@ func accountCreate(ctx *cli.Context) error { // accountUpdate transitions an account from a previous format to the current // one, also providing the possibility to change the pass-phrase. func accountUpdate(ctx *cli.Context) error { - if len(ctx.Args()) == 0 { + if ctx.Args().Len() == 0 { utils.Fatalf("No accounts specified to update") } stack, _ := makeConfigNode(ctx) ks := stack.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore) - for _, addr := range ctx.Args() { + for _, addr := range ctx.Args().Slice() { account, oldPassword := unlockAccount(ks, addr, 0, nil) newPassword := utils.GetPassPhraseWithList("Please give a new password. Do not forget this password.", true, 0, nil) if err := ks.Update(account, oldPassword, newPassword); err != nil { @@ -316,10 +314,10 @@ func accountUpdate(ctx *cli.Context) error { } func importWallet(ctx *cli.Context) error { - keyfile := ctx.Args().First() - if len(keyfile) == 0 { - utils.Fatalf("keyfile must be given as argument") + if ctx.Args().Len() != 1 { + utils.Fatalf("keyfile must be given as the only argument") } + keyfile := ctx.Args().First() keyJSON, err := os.ReadFile(keyfile) if err != nil { utils.Fatalf("Could not read wallet file: %v", err) @@ -338,10 +336,10 @@ func importWallet(ctx *cli.Context) error { } func accountImport(ctx *cli.Context) error { - keyfile := ctx.Args().First() - if len(keyfile) == 0 { - utils.Fatalf("keyfile must be given as argument") + if ctx.Args().Len() != 1 { + utils.Fatalf("keyfile must be given as the only argument") } + keyfile := ctx.Args().First() key, err := crypto.LoadECDSA(keyfile) if err != nil { utils.Fatalf("Failed to load the private key: %v", err) diff --git a/cmd/geth/accountcmd_test.go b/cmd/geth/accountcmd_test.go index 0c22e8c9bf576..84b9c33c24cf7 100644 --- a/cmd/geth/accountcmd_test.go +++ b/cmd/geth/accountcmd_test.go @@ -49,20 +49,27 @@ func TestAccountListEmpty(t *testing.T) { func TestAccountList(t *testing.T) { datadir := tmpDatadirWithKeystore(t) - geth := runGeth(t, "account", "list", "--datadir", datadir) - defer geth.ExpectExit() + var want = ` +Account #0: {7ef5a6135f1fd6a02593eedc869c6d41d934aef8} keystore://{{.Datadir}}/keystore/UTC--2016-03-22T12-57-55.920751759Z--7ef5a6135f1fd6a02593eedc869c6d41d934aef8 +Account #1: {f466859ead1932d743d622cb74fc058882e8648a} keystore://{{.Datadir}}/keystore/aaa +Account #2: {289d485d9771714cce91d3393d764e1311907acc} keystore://{{.Datadir}}/keystore/zzz +` if runtime.GOOS == "windows" { - geth.Expect(` + want = ` Account #0: {7ef5a6135f1fd6a02593eedc869c6d41d934aef8} keystore://{{.Datadir}}\keystore\UTC--2016-03-22T12-57-55.920751759Z--7ef5a6135f1fd6a02593eedc869c6d41d934aef8 Account #1: {f466859ead1932d743d622cb74fc058882e8648a} keystore://{{.Datadir}}\keystore\aaa Account #2: {289d485d9771714cce91d3393d764e1311907acc} keystore://{{.Datadir}}\keystore\zzz -`) - } else { - geth.Expect(` -Account #0: {7ef5a6135f1fd6a02593eedc869c6d41d934aef8} keystore://{{.Datadir}}/keystore/UTC--2016-03-22T12-57-55.920751759Z--7ef5a6135f1fd6a02593eedc869c6d41d934aef8 -Account #1: {f466859ead1932d743d622cb74fc058882e8648a} keystore://{{.Datadir}}/keystore/aaa -Account #2: {289d485d9771714cce91d3393d764e1311907acc} keystore://{{.Datadir}}/keystore/zzz -`) +` + } + { + geth := runGeth(t, "account", "list", "--datadir", datadir) + geth.Expect(want) + geth.ExpectExit() + } + { + geth := runGeth(t, "--datadir", datadir, "account", "list") + geth.Expect(want) + geth.ExpectExit() } } @@ -110,6 +117,20 @@ func TestAccountImport(t *testing.T) { } } +func TestAccountHelp(t *testing.T) { + geth := runGeth(t, "account", "-h") + geth.WaitExit() + if have, want := geth.ExitStatus(), 0; have != want { + t.Errorf("exit error, have %d want %d", have, want) + } + + geth = runGeth(t, "account", "import", "-h") + geth.WaitExit() + if have, want := geth.ExitStatus(), 0; have != want { + t.Errorf("exit error, have %d want %d", have, want) + } +} + func importAccountWithExpect(t *testing.T, key string, expected string) { dir := t.TempDir() keyfile := filepath.Join(dir, "key.prv") @@ -120,7 +141,7 @@ func importAccountWithExpect(t *testing.T, key string, expected string) { if err := os.WriteFile(passwordFile, []byte("foobar"), 0600); err != nil { t.Error(err) } - geth := runGeth(t, "--lightkdf", "account", "import", keyfile, "-password", passwordFile) + geth := runGeth(t, "--lightkdf", "account", "import", "-password", passwordFile, keyfile) defer geth.ExpectExit() geth.Expect(expected) } @@ -180,11 +201,12 @@ Fatal: could not decrypt key with given password func TestUnlockFlag(t *testing.T) { geth := runMinimalGeth(t, "--port", "0", "--ipcdisable", "--datadir", tmpDatadirWithKeystore(t), - "--unlock", "f466859ead1932d743d622cb74fc058882e8648a", "js", "testdata/empty.js") + "--unlock", "f466859ead1932d743d622cb74fc058882e8648a", "console", "--exec", "loadScript('testdata/empty.js')") geth.Expect(` Unlocking account f466859ead1932d743d622cb74fc058882e8648a | Attempt 1/3 !! Unsupported terminal, password will be echoed. Password: {{.InputLine "foobar"}} +undefined `) geth.ExpectExit() @@ -201,7 +223,7 @@ Password: {{.InputLine "foobar"}} func TestUnlockFlagWrongPassword(t *testing.T) { geth := runMinimalGeth(t, "--port", "0", "--ipcdisable", "--datadir", tmpDatadirWithKeystore(t), - "--unlock", "f466859ead1932d743d622cb74fc058882e8648a", "js", "testdata/empty.js") + "--unlock", "f466859ead1932d743d622cb74fc058882e8648a", "console", "--exec", "loadScript('testdata/empty.js')") defer geth.ExpectExit() geth.Expect(` @@ -219,7 +241,7 @@ Fatal: Failed to unlock account f466859ead1932d743d622cb74fc058882e8648a (could // https://github.com/ethereum/go-ethereum/issues/1785 func TestUnlockFlagMultiIndex(t *testing.T) { geth := runMinimalGeth(t, "--port", "0", "--ipcdisable", "--datadir", tmpDatadirWithKeystore(t), - "--unlock", "f466859ead1932d743d622cb74fc058882e8648a", "--unlock", "0,2", "js", "testdata/empty.js") + "--unlock", "f466859ead1932d743d622cb74fc058882e8648a", "--unlock", "0,2", "console", "--exec", "loadScript('testdata/empty.js')") geth.Expect(` Unlocking account 0 | Attempt 1/3 @@ -227,6 +249,7 @@ Unlocking account 0 | Attempt 1/3 Password: {{.InputLine "foobar"}} Unlocking account 2 | Attempt 1/3 Password: {{.InputLine "foobar"}} +undefined `) geth.ExpectExit() @@ -244,8 +267,11 @@ Password: {{.InputLine "foobar"}} func TestUnlockFlagPasswordFile(t *testing.T) { geth := runMinimalGeth(t, "--port", "0", "--ipcdisable", "--datadir", tmpDatadirWithKeystore(t), - "--unlock", "f466859ead1932d743d622cb74fc058882e8648a", "--password", "testdata/passwords.txt", "--unlock", "0,2", "js", "testdata/empty.js") + "--unlock", "f466859ead1932d743d622cb74fc058882e8648a", "--password", "testdata/passwords.txt", "--unlock", "0,2", "console", "--exec", "loadScript('testdata/empty.js')") + geth.Expect(` +undefined +`) geth.ExpectExit() wantMessages := []string{ @@ -275,7 +301,7 @@ func TestUnlockFlagAmbiguous(t *testing.T) { geth := runMinimalGeth(t, "--port", "0", "--ipcdisable", "--datadir", tmpDatadirWithKeystore(t), "--unlock", "f466859ead1932d743d622cb74fc058882e8648a", "--keystore", store, "--unlock", "f466859ead1932d743d622cb74fc058882e8648a", - "js", "testdata/empty.js") + "console", "--exec", "loadScript('testdata/empty.js')") defer geth.ExpectExit() // Helper for the expect template, returns absolute keystore path. @@ -294,6 +320,7 @@ Testing your password against all of them... Your password unlocked keystore://{{keypath "1"}} In order to avoid this warning, you need to remove the following duplicate key files: keystore://{{keypath "2"}} +undefined `) geth.ExpectExit() diff --git a/cmd/geth/attach_test.go b/cmd/geth/attach_test.go new file mode 100644 index 0000000000000..7c5f951750fbb --- /dev/null +++ b/cmd/geth/attach_test.go @@ -0,0 +1,83 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package main + +import ( + "fmt" + "net" + "net/http" + "sync/atomic" + "testing" +) + +type testHandler struct { + body func(http.ResponseWriter, *http.Request) +} + +func (t *testHandler) ServeHTTP(out http.ResponseWriter, in *http.Request) { + t.body(out, in) +} + +// TestAttachWithHeaders tests that 'geth attach' with custom headers works, i.e +// that custom headers are forwarded to the target. +func TestAttachWithHeaders(t *testing.T) { + t.Parallel() + ln, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatal(err) + } + port := ln.Addr().(*net.TCPAddr).Port + testReceiveHeaders(t, ln, "attach", "-H", "first: one", "-H", "second: two", fmt.Sprintf("http://localhost:%d", port)) + // This way to do it fails due to flag ordering: + // + // testReceiveHeaders(t, ln, "-H", "first: one", "-H", "second: two", "attach", fmt.Sprintf("http://localhost:%d", port)) + // This is fixed in a follow-up PR. +} + +// TestAttachWithHeaders tests that 'geth db --remotedb' with custom headers works, i.e +// that custom headers are forwarded to the target. +func TestRemoteDbWithHeaders(t *testing.T) { + t.Parallel() + ln, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatal(err) + } + port := ln.Addr().(*net.TCPAddr).Port + testReceiveHeaders(t, ln, "db", "metadata", "--remotedb", fmt.Sprintf("http://localhost:%d", port), "-H", "first: one", "-H", "second: two") +} + +func testReceiveHeaders(t *testing.T, ln net.Listener, gethArgs ...string) { + var ok uint32 + server := &http.Server{ + Addr: "localhost:0", + Handler: &testHandler{func(w http.ResponseWriter, r *http.Request) { + // We expect two headers + if have, want := r.Header.Get("first"), "one"; have != want { + t.Fatalf("missing header, have %v want %v", have, want) + } + if have, want := r.Header.Get("second"), "two"; have != want { + t.Fatalf("missing header, have %v want %v", have, want) + } + atomic.StoreUint32(&ok, 1) + }}} + go server.Serve(ln) + defer server.Close() + runGeth(t, gethArgs...).WaitExit() + if atomic.LoadUint32(&ok) != 1 { + t.Fatal("Test fail, expected invocation to succeed") + } +} diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index 6e19edeb46ebe..c89f736169ef0 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -35,20 +35,20 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/internal/flags" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/node" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) var ( - initCommand = cli.Command{ - Action: utils.MigrateFlags(initGenesis), + initCommand = &cli.Command{ + Action: initGenesis, Name: "init", Usage: "Bootstrap and initialize a new genesis block", ArgsUsage: "", Flags: utils.DatabasePathFlags, - Category: "BLOCKCHAIN COMMANDS", Description: ` The init command initializes a new genesis block and definition for the network. This is a destructive action and changes the network in which you will be @@ -56,22 +56,22 @@ participating. It expects the genesis file as argument.`, } - dumpGenesisCommand = cli.Command{ - Action: utils.MigrateFlags(dumpGenesis), + dumpGenesisCommand = &cli.Command{ + Action: dumpGenesis, Name: "dumpgenesis", Usage: "Dumps genesis block JSON configuration to stdout", ArgsUsage: "", - Flags: utils.NetworkFlags, - Category: "BLOCKCHAIN COMMANDS", + Flags: append([]cli.Flag{utils.DataDirFlag}, utils.NetworkFlags...), Description: ` -The dumpgenesis command dumps the genesis block configuration in JSON format to stdout.`, +The dumpgenesis command prints the genesis configuration of the network preset +if one is set. Otherwise it prints the genesis from the datadir.`, } - importCommand = cli.Command{ - Action: utils.MigrateFlags(importChain), + importCommand = &cli.Command{ + Action: importChain, Name: "import", Usage: "Import a blockchain file", ArgsUsage: " ( ... ) ", - Flags: append([]cli.Flag{ + Flags: flags.Merge([]cli.Flag{ utils.CacheFlag, utils.SyncModeFlag, utils.GCModeFlag, @@ -93,8 +93,7 @@ The dumpgenesis command dumps the genesis block configuration in JSON format to utils.MetricsInfluxDBBucketFlag, utils.MetricsInfluxDBOrganizationFlag, utils.TxLookupLimitFlag, - }, utils.DatabasePathFlags...), - Category: "BLOCKCHAIN COMMANDS", + }, utils.DatabasePathFlags), Description: ` The import command imports blocks from an RLP-encoded form. The form can be one file with several RLP-encoded blocks, or several files can be used. @@ -102,16 +101,15 @@ with several RLP-encoded blocks, or several files can be used. If only one file is used, import error will result in failure. If several files are used, processing will proceed even if an individual RLP-file import failure occurs.`, } - exportCommand = cli.Command{ - Action: utils.MigrateFlags(exportChain), + exportCommand = &cli.Command{ + Action: exportChain, Name: "export", Usage: "Export blockchain into file", ArgsUsage: " [ ]", - Flags: append([]cli.Flag{ + Flags: flags.Merge([]cli.Flag{ utils.CacheFlag, utils.SyncModeFlag, - }, utils.DatabasePathFlags...), - Category: "BLOCKCHAIN COMMANDS", + }, utils.DatabasePathFlags), Description: ` Requires a first argument of the file to write to. Optional second and third arguments control the first and @@ -119,42 +117,40 @@ last block to write. In this mode, the file will be appended if already existing. If the file ends with .gz, the output will be gzipped.`, } - importPreimagesCommand = cli.Command{ - Action: utils.MigrateFlags(importPreimages), + importPreimagesCommand = &cli.Command{ + Action: importPreimages, Name: "import-preimages", Usage: "Import the preimage database from an RLP stream", ArgsUsage: "", - Flags: append([]cli.Flag{ + Flags: flags.Merge([]cli.Flag{ utils.CacheFlag, utils.SyncModeFlag, - }, utils.DatabasePathFlags...), - Category: "BLOCKCHAIN COMMANDS", + }, utils.DatabasePathFlags), Description: ` The import-preimages command imports hash preimages from an RLP encoded stream. It's deprecated, please use "geth db import" instead. `, } - exportPreimagesCommand = cli.Command{ - Action: utils.MigrateFlags(exportPreimages), + exportPreimagesCommand = &cli.Command{ + Action: exportPreimages, Name: "export-preimages", Usage: "Export the preimage database into an RLP stream", ArgsUsage: "", - Flags: append([]cli.Flag{ + Flags: flags.Merge([]cli.Flag{ utils.CacheFlag, utils.SyncModeFlag, - }, utils.DatabasePathFlags...), - Category: "BLOCKCHAIN COMMANDS", + }, utils.DatabasePathFlags), Description: ` The export-preimages command exports hash preimages to an RLP encoded stream. It's deprecated, please use "geth db export" instead. `, } - dumpCommand = cli.Command{ - Action: utils.MigrateFlags(dump), + dumpCommand = &cli.Command{ + Action: dump, Name: "dump", Usage: "Dump a specific block from storage", ArgsUsage: "[? | ]", - Flags: append([]cli.Flag{ + Flags: flags.Merge([]cli.Flag{ utils.CacheFlag, utils.IterativeOutputFlag, utils.ExcludeCodeFlag, @@ -162,8 +158,7 @@ It's deprecated, please use "geth db export" instead. utils.IncludeIncompletesFlag, utils.StartKeyFlag, utils.DumpLimitFlag, - }, utils.DatabasePathFlags...), - Category: "BLOCKCHAIN COMMANDS", + }, utils.DatabasePathFlags), Description: ` This command dumps out the state for a given block (or latest, if none provided). `, @@ -173,10 +168,12 @@ This command dumps out the state for a given block (or latest, if none provided) // initGenesis will initialise the given JSON format genesis file and writes it as // the zero'd block (i.e. genesis) or will fail hard if it can't succeed. func initGenesis(ctx *cli.Context) error { - // Make sure we have a valid genesis JSON + if ctx.Args().Len() != 1 { + utils.Fatalf("need genesis.json file as the only argument") + } genesisPath := ctx.Args().First() if len(genesisPath) == 0 { - utils.Fatalf("Must supply path to genesis JSON file") + utils.Fatalf("invalid path to genesis file") } file, err := os.Open(genesisPath) if err != nil { @@ -192,7 +189,7 @@ func initGenesis(ctx *cli.Context) error { stack, _ := makeConfigNode(ctx) defer stack.Close() for _, name := range []string{"chaindata", "lightchaindata"} { - chaindb, err := stack.OpenDatabaseWithFreezer(name, 0, 0, ctx.GlobalString(utils.AncientFlag.Name), "", false) + chaindb, err := stack.OpenDatabaseWithFreezer(name, 0, 0, ctx.String(utils.AncientFlag.Name), "", false) if err != nil { utils.Fatalf("Failed to open database: %v", err) } @@ -207,19 +204,44 @@ func initGenesis(ctx *cli.Context) error { } func dumpGenesis(ctx *cli.Context) error { - // TODO(rjl493456442) support loading from the custom datadir - genesis := utils.MakeGenesis(ctx) - if genesis == nil { - genesis = core.DefaultGenesisBlock() + // if there is a testnet preset enabled, dump that + if utils.IsNetworkPreset(ctx) { + genesis := utils.MakeGenesis(ctx) + if err := json.NewEncoder(os.Stdout).Encode(genesis); err != nil { + utils.Fatalf("could not encode genesis: %s", err) + } + return nil + } + // dump whatever already exists in the datadir + stack, _ := makeConfigNode(ctx) + for _, name := range []string{"chaindata", "lightchaindata"} { + db, err := stack.OpenDatabase(name, 0, 0, "", true) + if err != nil { + if !os.IsNotExist(err) { + return err + } + continue + } + genesis, err := core.ReadGenesis(db) + if err != nil { + utils.Fatalf("failed to read genesis: %s", err) + } + db.Close() + + if err := json.NewEncoder(os.Stdout).Encode(*genesis); err != nil { + utils.Fatalf("could not encode stored genesis: %s", err) + } + return nil } - if err := json.NewEncoder(os.Stdout).Encode(genesis); err != nil { - utils.Fatalf("could not encode genesis") + if ctx.IsSet(utils.DataDirFlag.Name) { + utils.Fatalf("no existing datadir at %s", stack.Config().DataDir) } + utils.Fatalf("no network preset provided. no exisiting genesis in the default datadir") return nil } func importChain(ctx *cli.Context) error { - if len(ctx.Args()) < 1 { + if ctx.Args().Len() < 1 { utils.Fatalf("This command requires an argument.") } // Start metrics export if enabled @@ -253,13 +275,13 @@ func importChain(ctx *cli.Context) error { var importErr error - if len(ctx.Args()) == 1 { + if ctx.Args().Len() == 1 { if err := utils.ImportChain(chain, ctx.Args().First()); err != nil { importErr = err log.Error("Import error", "err", err) } } else { - for _, arg := range ctx.Args() { + for _, arg := range ctx.Args().Slice() { if err := utils.ImportChain(chain, arg); err != nil { importErr = err log.Error("Import error", "file", arg, "err", err) @@ -281,7 +303,7 @@ func importChain(ctx *cli.Context) error { fmt.Printf("Allocations: %.3f million\n", float64(mem.Mallocs)/1000000) fmt.Printf("GC pause: %v\n\n", time.Duration(mem.PauseTotalNs)) - if ctx.GlobalBool(utils.NoCompactionFlag.Name) { + if ctx.Bool(utils.NoCompactionFlag.Name) { return nil } @@ -298,7 +320,7 @@ func importChain(ctx *cli.Context) error { } func exportChain(ctx *cli.Context) error { - if len(ctx.Args()) < 1 { + if ctx.Args().Len() < 1 { utils.Fatalf("This command requires an argument.") } @@ -310,7 +332,7 @@ func exportChain(ctx *cli.Context) error { var err error fp := ctx.Args().First() - if len(ctx.Args()) < 3 { + if ctx.Args().Len() < 3 { err = utils.ExportChain(chain, fp) } else { // This can be improved to allow for numbers larger than 9223372036854775807 @@ -337,7 +359,7 @@ func exportChain(ctx *cli.Context) error { // importPreimages imports preimage data from the specified file. func importPreimages(ctx *cli.Context) error { - if len(ctx.Args()) < 1 { + if ctx.Args().Len() < 1 { utils.Fatalf("This command requires an argument.") } @@ -356,7 +378,7 @@ func importPreimages(ctx *cli.Context) error { // exportPreimages dumps the preimage data to specified json file in streaming way. func exportPreimages(ctx *cli.Context) error { - if len(ctx.Args()) < 1 { + if ctx.Args().Len() < 1 { utils.Fatalf("This command requires an argument.") } stack, _ := makeConfigNode(ctx) @@ -388,12 +410,12 @@ func parseDumpConfig(ctx *cli.Context, stack *node.Node) (*state.DumpConfig, eth return nil, nil, common.Hash{}, fmt.Errorf("block %x not found", hash) } } else { - number, err := strconv.Atoi(arg) + number, err := strconv.ParseUint(arg, 10, 64) if err != nil { return nil, nil, common.Hash{}, err } - if hash := rawdb.ReadCanonicalHash(db, uint64(number)); hash != (common.Hash{}) { - header = rawdb.ReadHeader(db, hash, uint64(number)) + if hash := rawdb.ReadCanonicalHash(db, number); hash != (common.Hash{}) { + header = rawdb.ReadHeader(db, hash, number) } else { return nil, nil, common.Hash{}, fmt.Errorf("header for block %d not found", number) } diff --git a/cmd/geth/config.go b/cmd/geth/config.go index 6d2bb2bcb6226..a8cee0d13a59b 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -20,12 +20,11 @@ import ( "bufio" "errors" "fmt" - "math/big" "os" "reflect" "unicode" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" "github.com/ethereum/go-ethereum/accounts/external" "github.com/ethereum/go-ethereum/accounts/keystore" @@ -35,6 +34,8 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/internal/ethapi" + "github.com/ethereum/go-ethereum/internal/flags" + "github.com/ethereum/go-ethereum/internal/version" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/node" @@ -43,19 +44,19 @@ import ( ) var ( - dumpConfigCommand = cli.Command{ - Action: utils.MigrateFlags(dumpConfig), + dumpConfigCommand = &cli.Command{ + Action: dumpConfig, Name: "dumpconfig", Usage: "Show configuration values", ArgsUsage: "", - Flags: utils.GroupFlags(nodeFlags, rpcFlags), - Category: "MISCELLANEOUS COMMANDS", + Flags: flags.Merge(nodeFlags, rpcFlags), Description: `The dumpconfig command shows configuration values.`, } - configFileFlag = cli.StringFlag{ - Name: "config", - Usage: "TOML configuration file", + configFileFlag = &cli.StringFlag{ + Name: "config", + Usage: "TOML configuration file", + Category: flags.EthCategory, } ) @@ -108,9 +109,10 @@ func loadConfig(file string, cfg *gethConfig) error { } func defaultNodeConfig() node.Config { + git, _ := version.VCS() cfg := node.DefaultConfig cfg.Name = clientIdentifier - cfg.Version = params.VersionWithCommit(gitCommit, gitDate) + cfg.Version = params.VersionWithCommit(git.Commit, git.Date) cfg.HTTPModules = append(cfg.HTTPModules, "eth") cfg.WSModules = append(cfg.WSModules, "eth") cfg.IPCPath = "geth.ipc" @@ -127,7 +129,7 @@ func makeConfigNode(ctx *cli.Context) (*node.Node, gethConfig) { } // Load config file. - if file := ctx.GlobalString(configFileFlag.Name); file != "" { + if file := ctx.String(configFileFlag.Name); file != "" { if err := loadConfig(file, &cfg); err != nil { utils.Fatalf("%v", err) } @@ -145,8 +147,8 @@ func makeConfigNode(ctx *cli.Context) (*node.Node, gethConfig) { } utils.SetEthConfig(ctx, stack, &cfg.Eth) - if ctx.GlobalIsSet(utils.EthStatsURLFlag.Name) { - cfg.Ethstats.URL = ctx.GlobalString(utils.EthStatsURLFlag.Name) + if ctx.IsSet(utils.EthStatsURLFlag.Name) { + cfg.Ethstats.URL = ctx.String(utils.EthStatsURLFlag.Name) } applyMetricConfig(ctx, &cfg) @@ -156,15 +158,18 @@ func makeConfigNode(ctx *cli.Context) (*node.Node, gethConfig) { // makeFullNode loads geth configuration and creates the Ethereum backend. func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) { stack, cfg := makeConfigNode(ctx) - if ctx.GlobalIsSet(utils.OverrideArrowGlacierFlag.Name) { - cfg.Eth.OverrideArrowGlacier = new(big.Int).SetUint64(ctx.GlobalUint64(utils.OverrideArrowGlacierFlag.Name)) + if ctx.IsSet(utils.OverrideTerminalTotalDifficulty.Name) { + cfg.Eth.OverrideTerminalTotalDifficulty = flags.GlobalBig(ctx, utils.OverrideTerminalTotalDifficulty.Name) } - if ctx.GlobalIsSet(utils.OverrideTerminalTotalDifficulty.Name) { - cfg.Eth.OverrideTerminalTotalDifficulty = utils.GlobalBig(ctx, utils.OverrideTerminalTotalDifficulty.Name) + if ctx.IsSet(utils.OverrideTerminalTotalDifficultyPassed.Name) { + override := ctx.Bool(utils.OverrideTerminalTotalDifficultyPassed.Name) + cfg.Eth.OverrideTerminalTotalDifficultyPassed = &override } + backend, eth := utils.RegisterEthService(stack, &cfg.Eth) + // Warn users to migrate if they have a legacy freezer format. - if eth != nil && !ctx.GlobalIsSet(utils.IgnoreLegacyReceiptsFlag.Name) { + if eth != nil && !ctx.IsSet(utils.IgnoreLegacyReceiptsFlag.Name) { firstIdx := uint64(0) // Hack to speed up check for mainnet because we know // the first non-empty block. @@ -172,19 +177,24 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) { if cfg.Eth.NetworkId == 1 && ghash == params.MainnetGenesisHash { firstIdx = 46147 } - isLegacy, _, err := dbHasLegacyReceipts(eth.ChainDb(), firstIdx) + isLegacy, firstLegacy, err := dbHasLegacyReceipts(eth.ChainDb(), firstIdx) if err != nil { log.Error("Failed to check db for legacy receipts", "err", err) } else if isLegacy { stack.Close() - utils.Fatalf("Database has receipts with a legacy format. Please run `geth db freezer-migrate`.") + log.Error("Database has receipts with a legacy format", "firstLegacy", firstLegacy) + utils.Fatalf("Aborting. Please run `geth db freezer-migrate`.") } } - // Configure GraphQL if requested - if ctx.GlobalIsSet(utils.GraphQLEnabledFlag.Name) { - utils.RegisterGraphQLService(stack, backend, cfg.Node) + // Configure log filter RPC API. + filterSystem := utils.RegisterFilterAPI(stack, backend, &cfg.Eth) + + // Configure GraphQL if requested. + if ctx.IsSet(utils.GraphQLEnabledFlag.Name) { + utils.RegisterGraphQLService(stack, backend, filterSystem, &cfg.Node) } + // Add the Ethereum Stats daemon if requested. if cfg.Ethstats.URL != "" { utils.RegisterEthStatsService(stack, backend, cfg.Ethstats.URL) @@ -222,47 +232,47 @@ func dumpConfig(ctx *cli.Context) error { } func applyMetricConfig(ctx *cli.Context, cfg *gethConfig) { - if ctx.GlobalIsSet(utils.MetricsEnabledFlag.Name) { - cfg.Metrics.Enabled = ctx.GlobalBool(utils.MetricsEnabledFlag.Name) + if ctx.IsSet(utils.MetricsEnabledFlag.Name) { + cfg.Metrics.Enabled = ctx.Bool(utils.MetricsEnabledFlag.Name) } - if ctx.GlobalIsSet(utils.MetricsEnabledExpensiveFlag.Name) { - cfg.Metrics.EnabledExpensive = ctx.GlobalBool(utils.MetricsEnabledExpensiveFlag.Name) + if ctx.IsSet(utils.MetricsEnabledExpensiveFlag.Name) { + cfg.Metrics.EnabledExpensive = ctx.Bool(utils.MetricsEnabledExpensiveFlag.Name) } - if ctx.GlobalIsSet(utils.MetricsHTTPFlag.Name) { - cfg.Metrics.HTTP = ctx.GlobalString(utils.MetricsHTTPFlag.Name) + if ctx.IsSet(utils.MetricsHTTPFlag.Name) { + cfg.Metrics.HTTP = ctx.String(utils.MetricsHTTPFlag.Name) } - if ctx.GlobalIsSet(utils.MetricsPortFlag.Name) { - cfg.Metrics.Port = ctx.GlobalInt(utils.MetricsPortFlag.Name) + if ctx.IsSet(utils.MetricsPortFlag.Name) { + cfg.Metrics.Port = ctx.Int(utils.MetricsPortFlag.Name) } - if ctx.GlobalIsSet(utils.MetricsEnableInfluxDBFlag.Name) { - cfg.Metrics.EnableInfluxDB = ctx.GlobalBool(utils.MetricsEnableInfluxDBFlag.Name) + if ctx.IsSet(utils.MetricsEnableInfluxDBFlag.Name) { + cfg.Metrics.EnableInfluxDB = ctx.Bool(utils.MetricsEnableInfluxDBFlag.Name) } - if ctx.GlobalIsSet(utils.MetricsInfluxDBEndpointFlag.Name) { - cfg.Metrics.InfluxDBEndpoint = ctx.GlobalString(utils.MetricsInfluxDBEndpointFlag.Name) + if ctx.IsSet(utils.MetricsInfluxDBEndpointFlag.Name) { + cfg.Metrics.InfluxDBEndpoint = ctx.String(utils.MetricsInfluxDBEndpointFlag.Name) } - if ctx.GlobalIsSet(utils.MetricsInfluxDBDatabaseFlag.Name) { - cfg.Metrics.InfluxDBDatabase = ctx.GlobalString(utils.MetricsInfluxDBDatabaseFlag.Name) + if ctx.IsSet(utils.MetricsInfluxDBDatabaseFlag.Name) { + cfg.Metrics.InfluxDBDatabase = ctx.String(utils.MetricsInfluxDBDatabaseFlag.Name) } - if ctx.GlobalIsSet(utils.MetricsInfluxDBUsernameFlag.Name) { - cfg.Metrics.InfluxDBUsername = ctx.GlobalString(utils.MetricsInfluxDBUsernameFlag.Name) + if ctx.IsSet(utils.MetricsInfluxDBUsernameFlag.Name) { + cfg.Metrics.InfluxDBUsername = ctx.String(utils.MetricsInfluxDBUsernameFlag.Name) } - if ctx.GlobalIsSet(utils.MetricsInfluxDBPasswordFlag.Name) { - cfg.Metrics.InfluxDBPassword = ctx.GlobalString(utils.MetricsInfluxDBPasswordFlag.Name) + if ctx.IsSet(utils.MetricsInfluxDBPasswordFlag.Name) { + cfg.Metrics.InfluxDBPassword = ctx.String(utils.MetricsInfluxDBPasswordFlag.Name) } - if ctx.GlobalIsSet(utils.MetricsInfluxDBTagsFlag.Name) { - cfg.Metrics.InfluxDBTags = ctx.GlobalString(utils.MetricsInfluxDBTagsFlag.Name) + if ctx.IsSet(utils.MetricsInfluxDBTagsFlag.Name) { + cfg.Metrics.InfluxDBTags = ctx.String(utils.MetricsInfluxDBTagsFlag.Name) } - if ctx.GlobalIsSet(utils.MetricsEnableInfluxDBV2Flag.Name) { - cfg.Metrics.EnableInfluxDBV2 = ctx.GlobalBool(utils.MetricsEnableInfluxDBV2Flag.Name) + if ctx.IsSet(utils.MetricsEnableInfluxDBV2Flag.Name) { + cfg.Metrics.EnableInfluxDBV2 = ctx.Bool(utils.MetricsEnableInfluxDBV2Flag.Name) } - if ctx.GlobalIsSet(utils.MetricsInfluxDBTokenFlag.Name) { - cfg.Metrics.InfluxDBToken = ctx.GlobalString(utils.MetricsInfluxDBTokenFlag.Name) + if ctx.IsSet(utils.MetricsInfluxDBTokenFlag.Name) { + cfg.Metrics.InfluxDBToken = ctx.String(utils.MetricsInfluxDBTokenFlag.Name) } - if ctx.GlobalIsSet(utils.MetricsInfluxDBBucketFlag.Name) { - cfg.Metrics.InfluxDBBucket = ctx.GlobalString(utils.MetricsInfluxDBBucketFlag.Name) + if ctx.IsSet(utils.MetricsInfluxDBBucketFlag.Name) { + cfg.Metrics.InfluxDBBucket = ctx.String(utils.MetricsInfluxDBBucketFlag.Name) } - if ctx.GlobalIsSet(utils.MetricsInfluxDBOrganizationFlag.Name) { - cfg.Metrics.InfluxDBOrganization = ctx.GlobalString(utils.MetricsInfluxDBOrganizationFlag.Name) + if ctx.IsSet(utils.MetricsInfluxDBOrganizationFlag.Name) { + cfg.Metrics.InfluxDBOrganization = ctx.String(utils.MetricsInfluxDBOrganizationFlag.Name) } } diff --git a/cmd/geth/consolecmd.go b/cmd/geth/consolecmd.go index 5167f8536a277..83c6b66a8a604 100644 --- a/cmd/geth/consolecmd.go +++ b/cmd/geth/consolecmd.go @@ -18,39 +18,34 @@ package main import ( "fmt" - "path/filepath" "strings" "github.com/ethereum/go-ethereum/cmd/utils" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/console" - "github.com/ethereum/go-ethereum/node" - "github.com/ethereum/go-ethereum/rpc" - "gopkg.in/urfave/cli.v1" + "github.com/ethereum/go-ethereum/internal/flags" + "github.com/urfave/cli/v2" ) var ( consoleFlags = []cli.Flag{utils.JSpathFlag, utils.ExecFlag, utils.PreloadJSFlag} - consoleCommand = cli.Command{ - Action: utils.MigrateFlags(localConsole), - Name: "console", - Usage: "Start an interactive JavaScript environment", - Flags: utils.GroupFlags(nodeFlags, rpcFlags, consoleFlags), - Category: "CONSOLE COMMANDS", + consoleCommand = &cli.Command{ + Action: localConsole, + Name: "console", + Usage: "Start an interactive JavaScript environment", + Flags: flags.Merge(nodeFlags, rpcFlags, consoleFlags), Description: ` The Geth console is an interactive shell for the JavaScript runtime environment which exposes a node admin interface as well as the Ðapp JavaScript API. See https://geth.ethereum.org/docs/interface/javascript-console.`, } - attachCommand = cli.Command{ - Action: utils.MigrateFlags(remoteConsole), + attachCommand = &cli.Command{ + Action: remoteConsole, Name: "attach", Usage: "Start an interactive JavaScript environment (connect to node)", ArgsUsage: "[endpoint]", - Flags: utils.GroupFlags([]cli.Flag{utils.DataDirFlag}, consoleFlags), - Category: "CONSOLE COMMANDS", + Flags: flags.Merge([]cli.Flag{utils.DataDirFlag, utils.HttpHeaderFlag}, consoleFlags), Description: ` The Geth console is an interactive shell for the JavaScript runtime environment which exposes a node admin interface as well as the Ðapp JavaScript API. @@ -58,13 +53,12 @@ See https://geth.ethereum.org/docs/interface/javascript-console. This command allows to open a console on a running geth node.`, } - javascriptCommand = cli.Command{ - Action: utils.MigrateFlags(ephemeralConsole), + javascriptCommand = &cli.Command{ + Action: ephemeralConsole, Name: "js", - Usage: "Execute the specified JavaScript files", + Usage: "(DEPRECATED) Execute the specified JavaScript files", ArgsUsage: " [jsfile...]", - Flags: utils.GroupFlags(nodeFlags, consoleFlags), - Category: "CONSOLE COMMANDS", + Flags: flags.Merge(nodeFlags, consoleFlags), Description: ` The JavaScript VM exposes a node admin interface as well as the Ðapp JavaScript API. See https://geth.ethereum.org/docs/interface/javascript-console`, @@ -87,7 +81,7 @@ func localConsole(ctx *cli.Context) error { } config := console.Config{ DataDir: utils.MakeDataDir(ctx), - DocRoot: ctx.GlobalString(utils.JSpathFlag.Name), + DocRoot: ctx.String(utils.JSpathFlag.Name), Client: client, Preload: utils.MakeConsolePreloads(ctx), } @@ -98,7 +92,7 @@ func localConsole(ctx *cli.Context) error { defer console.Stop(false) // If only a short execution was requested, evaluate and return. - if script := ctx.GlobalString(utils.ExecFlag.Name); script != "" { + if script := ctx.String(utils.ExecFlag.Name); script != "" { console.Evaluate(script) return nil } @@ -119,41 +113,22 @@ func localConsole(ctx *cli.Context) error { // remoteConsole will connect to a remote geth instance, attaching a JavaScript // console to it. func remoteConsole(ctx *cli.Context) error { + if ctx.Args().Len() > 1 { + utils.Fatalf("invalid command-line: too many arguments") + } endpoint := ctx.Args().First() if endpoint == "" { - path := node.DefaultDataDir() - if ctx.GlobalIsSet(utils.DataDirFlag.Name) { - path = ctx.GlobalString(utils.DataDirFlag.Name) - } - if path != "" { - if ctx.GlobalBool(utils.RopstenFlag.Name) { - // Maintain compatibility with older Geth configurations storing the - // Ropsten database in `testnet` instead of `ropsten`. - legacyPath := filepath.Join(path, "testnet") - if common.FileExist(legacyPath) { - path = legacyPath - } else { - path = filepath.Join(path, "ropsten") - } - } else if ctx.GlobalBool(utils.RinkebyFlag.Name) { - path = filepath.Join(path, "rinkeby") - } else if ctx.GlobalBool(utils.GoerliFlag.Name) { - path = filepath.Join(path, "goerli") - } else if ctx.GlobalBool(utils.SepoliaFlag.Name) { - path = filepath.Join(path, "sepolia") - } else if ctx.GlobalBool(utils.KilnFlag.Name) { - path = filepath.Join(path, "kiln") - } - } - endpoint = fmt.Sprintf("%s/geth.ipc", path) + cfg := defaultNodeConfig() + utils.SetDataDir(ctx, &cfg) + endpoint = cfg.IPCEndpoint() } - client, err := dialRPC(endpoint) + client, err := utils.DialRPCWithHeaders(endpoint, ctx.StringSlice(utils.HttpHeaderFlag.Name)) if err != nil { utils.Fatalf("Unable to attach to remote geth: %v", err) } config := console.Config{ DataDir: utils.MakeDataDir(ctx), - DocRoot: ctx.GlobalString(utils.JSpathFlag.Name), + DocRoot: ctx.String(utils.JSpathFlag.Name), Client: client, Preload: utils.MakeConsolePreloads(ctx), } @@ -163,7 +138,7 @@ func remoteConsole(ctx *cli.Context) error { } defer console.Stop(false) - if script := ctx.GlobalString(utils.ExecFlag.Name); script != "" { + if script := ctx.String(utils.ExecFlag.Name); script != "" { console.Evaluate(script) return nil } @@ -174,61 +149,15 @@ func remoteConsole(ctx *cli.Context) error { return nil } -// dialRPC returns a RPC client which connects to the given endpoint. -// The check for empty endpoint implements the defaulting logic -// for "geth attach" with no argument. -func dialRPC(endpoint string) (*rpc.Client, error) { - if endpoint == "" { - endpoint = node.DefaultIPCEndpoint(clientIdentifier) - } else if strings.HasPrefix(endpoint, "rpc:") || strings.HasPrefix(endpoint, "ipc:") { - // Backwards compatibility with geth < 1.5 which required - // these prefixes. - endpoint = endpoint[4:] - } - return rpc.Dial(endpoint) -} - // ephemeralConsole starts a new geth node, attaches an ephemeral JavaScript // console to it, executes each of the files specified as arguments and tears // everything down. func ephemeralConsole(ctx *cli.Context) error { - // Create and start the node based on the CLI flags - stack, backend := makeFullNode(ctx) - startNode(ctx, stack, backend, false) - defer stack.Close() - - // Attach to the newly started node and start the JavaScript console - client, err := stack.Attach() - if err != nil { - return fmt.Errorf("Failed to attach to the inproc geth: %v", err) - } - config := console.Config{ - DataDir: utils.MakeDataDir(ctx), - DocRoot: ctx.GlobalString(utils.JSpathFlag.Name), - Client: client, - Preload: utils.MakeConsolePreloads(ctx), - } - - console, err := console.New(config) - if err != nil { - return fmt.Errorf("Failed to start the JavaScript console: %v", err) + var b strings.Builder + for _, file := range ctx.Args().Slice() { + b.Write([]byte(fmt.Sprintf("loadScript('%s');", file))) } - defer console.Stop(false) - - // Interrupt the JS interpreter when node is stopped. - go func() { - stack.Wait() - console.Stop(false) - }() - - // Evaluate each of the specified JavaScript files. - for _, file := range ctx.Args() { - if err = console.Execute(file); err != nil { - return fmt.Errorf("Failed to execute %s: %v", file, err) - } - } - - // The main script is now done, but keep running timers/callbacks. - console.Stop(true) + utils.Fatalf(`The "js" command is deprecated. Please use the following instead: +geth --exec "%s" console`, b.String()) return nil } diff --git a/cmd/geth/consolecmd_test.go b/cmd/geth/consolecmd_test.go index f4e8bf490a32e..442b82df0b3ce 100644 --- a/cmd/geth/consolecmd_test.go +++ b/cmd/geth/consolecmd_test.go @@ -41,7 +41,7 @@ func runMinimalGeth(t *testing.T, args ...string) *testgeth { // --ropsten to make the 'writing genesis to disk' faster (no accounts) // --networkid=1337 to avoid cache bump // --syncmode=full to avoid allocating fast sync bloom - allArgs := []string{"--ropsten", "--networkid", "1337", "--syncmode=full", "--port", "0", + allArgs := []string{"--ropsten", "--networkid", "1337", "--authrpc.port", "0", "--syncmode=full", "--port", "0", "--nat", "none", "--nodiscover", "--maxpeers", "0", "--cache", "64", "--datadir.minfreedisk", "0"} return runGeth(t, append(allArgs, args...)...) @@ -155,7 +155,7 @@ To exit, press ctrl-d or type exit } // trulyRandInt generates a crypto random integer used by the console tests to -// not clash network ports with other tests running cocurrently. +// not clash network ports with other tests running concurrently. func trulyRandInt(lo, hi int) int { num, _ := rand.Int(rand.Reader, big.NewInt(int64(hi-lo))) return int(num.Int64()) + lo diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index c4fe9251f9b8b..9d834ee14b9dd 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -22,7 +22,6 @@ import ( "os" "os/signal" "path/filepath" - "sort" "strconv" "strings" "syscall" @@ -37,29 +36,28 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/internal/flags" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/trie" "github.com/olekukonko/tablewriter" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) var ( - removedbCommand = cli.Command{ - Action: utils.MigrateFlags(removeDB), + removedbCommand = &cli.Command{ + Action: removeDB, Name: "removedb", Usage: "Remove blockchain and state databases", ArgsUsage: "", Flags: utils.DatabasePathFlags, - Category: "DATABASE COMMANDS", Description: ` Remove blockchain and state databases`, } - dbCommand = cli.Command{ + dbCommand = &cli.Command{ Name: "db", Usage: "Low level database operations", ArgsUsage: "", - Category: "DATABASE COMMANDS", - Subcommands: []cli.Command{ + Subcommands: []*cli.Command{ dbInspectCmd, dbStatCmd, dbCompactCmd, @@ -75,39 +73,39 @@ Remove blockchain and state databases`, dbCheckStateContentCmd, }, } - dbInspectCmd = cli.Command{ - Action: utils.MigrateFlags(inspect), + dbInspectCmd = &cli.Command{ + Action: inspect, Name: "inspect", ArgsUsage: " ", - Flags: utils.GroupFlags([]cli.Flag{ + Flags: flags.Merge([]cli.Flag{ utils.SyncModeFlag, }, utils.NetworkFlags, utils.DatabasePathFlags), Usage: "Inspect the storage size for each type of data in the database", Description: `This commands iterates the entire database. If the optional 'prefix' and 'start' arguments are provided, then the iteration is limited to the given subset of data.`, } - dbCheckStateContentCmd = cli.Command{ - Action: utils.MigrateFlags(checkStateContent), + dbCheckStateContentCmd = &cli.Command{ + Action: checkStateContent, Name: "check-state-content", ArgsUsage: "", - Flags: utils.GroupFlags(utils.NetworkFlags, utils.DatabasePathFlags), + Flags: flags.Merge(utils.NetworkFlags, utils.DatabasePathFlags), Usage: "Verify that state data is cryptographically correct", Description: `This command iterates the entire database for 32-byte keys, looking for rlp-encoded trie nodes. For each trie node encountered, it checks that the key corresponds to the keccak256(value). If this is not true, this indicates a data corruption.`, } - dbStatCmd = cli.Command{ - Action: utils.MigrateFlags(dbStats), + dbStatCmd = &cli.Command{ + Action: dbStats, Name: "stats", Usage: "Print leveldb statistics", - Flags: utils.GroupFlags([]cli.Flag{ + Flags: flags.Merge([]cli.Flag{ utils.SyncModeFlag, }, utils.NetworkFlags, utils.DatabasePathFlags), } - dbCompactCmd = cli.Command{ - Action: utils.MigrateFlags(dbCompact), + dbCompactCmd = &cli.Command{ + Action: dbCompact, Name: "compact", Usage: "Compact leveldb database. WARNING: May take a very long time", - Flags: utils.GroupFlags([]cli.Flag{ + Flags: flags.Merge([]cli.Flag{ utils.SyncModeFlag, utils.CacheFlag, utils.CacheDatabaseFlag, @@ -116,93 +114,93 @@ a data corruption.`, WARNING: This operation may take a very long time to finish, and may cause database corruption if it is aborted during execution'!`, } - dbGetCmd = cli.Command{ - Action: utils.MigrateFlags(dbGet), + dbGetCmd = &cli.Command{ + Action: dbGet, Name: "get", Usage: "Show the value of a database key", ArgsUsage: "", - Flags: utils.GroupFlags([]cli.Flag{ + Flags: flags.Merge([]cli.Flag{ utils.SyncModeFlag, }, utils.NetworkFlags, utils.DatabasePathFlags), Description: "This command looks up the specified database key from the database.", } - dbDeleteCmd = cli.Command{ - Action: utils.MigrateFlags(dbDelete), + dbDeleteCmd = &cli.Command{ + Action: dbDelete, Name: "delete", Usage: "Delete a database key (WARNING: may corrupt your database)", ArgsUsage: "", - Flags: utils.GroupFlags([]cli.Flag{ + Flags: flags.Merge([]cli.Flag{ utils.SyncModeFlag, }, utils.NetworkFlags, utils.DatabasePathFlags), Description: `This command deletes the specified database key from the database. WARNING: This is a low-level operation which may cause database corruption!`, } - dbPutCmd = cli.Command{ - Action: utils.MigrateFlags(dbPut), + dbPutCmd = &cli.Command{ + Action: dbPut, Name: "put", Usage: "Set the value of a database key (WARNING: may corrupt your database)", ArgsUsage: " ", - Flags: utils.GroupFlags([]cli.Flag{ + Flags: flags.Merge([]cli.Flag{ utils.SyncModeFlag, }, utils.NetworkFlags, utils.DatabasePathFlags), Description: `This command sets a given database key to the given value. WARNING: This is a low-level operation which may cause database corruption!`, } - dbGetSlotsCmd = cli.Command{ - Action: utils.MigrateFlags(dbDumpTrie), + dbGetSlotsCmd = &cli.Command{ + Action: dbDumpTrie, Name: "dumptrie", Usage: "Show the storage key/values of a given storage trie", - ArgsUsage: " ", - Flags: utils.GroupFlags([]cli.Flag{ + ArgsUsage: " ", + Flags: flags.Merge([]cli.Flag{ utils.SyncModeFlag, }, utils.NetworkFlags, utils.DatabasePathFlags), Description: "This command looks up the specified database key from the database.", } - dbDumpFreezerIndex = cli.Command{ - Action: utils.MigrateFlags(freezerInspect), + dbDumpFreezerIndex = &cli.Command{ + Action: freezerInspect, Name: "freezer-index", - Usage: "Dump out the index of a given freezer type", - ArgsUsage: " ", - Flags: utils.GroupFlags([]cli.Flag{ + Usage: "Dump out the index of a specific freezer table", + ArgsUsage: " ", + Flags: flags.Merge([]cli.Flag{ utils.SyncModeFlag, }, utils.NetworkFlags, utils.DatabasePathFlags), Description: "This command displays information about the freezer index.", } - dbImportCmd = cli.Command{ - Action: utils.MigrateFlags(importLDBdata), + dbImportCmd = &cli.Command{ + Action: importLDBdata, Name: "import", Usage: "Imports leveldb-data from an exported RLP dump.", ArgsUsage: " has .gz suffix, gzip compression will be used.", ArgsUsage: " ", - Flags: utils.GroupFlags([]cli.Flag{ + Flags: flags.Merge([]cli.Flag{ utils.SyncModeFlag, }, utils.NetworkFlags, utils.DatabasePathFlags), Description: "Exports the specified chain data to an RLP encoded stream, optionally gzip-compressed.", } - dbMetadataCmd = cli.Command{ - Action: utils.MigrateFlags(showMetaData), + dbMetadataCmd = &cli.Command{ + Action: showMetaData, Name: "metadata", Usage: "Shows metadata about the chain status.", - Flags: utils.GroupFlags([]cli.Flag{ + Flags: flags.Merge([]cli.Flag{ utils.SyncModeFlag, }, utils.NetworkFlags, utils.DatabasePathFlags), Description: "Shows metadata about the chain status.", } - dbMigrateFreezerCmd = cli.Command{ - Action: utils.MigrateFlags(freezerMigrate), + dbMigrateFreezerCmd = &cli.Command{ + Action: freezerMigrate, Name: "freezer-migrate", Usage: "Migrate legacy parts of the freezer. (WARNING: may take a long time)", ArgsUsage: "", - Flags: utils.GroupFlags([]cli.Flag{ + Flags: flags.Merge([]cli.Flag{ utils.SyncModeFlag, }, utils.NetworkFlags, utils.DatabasePathFlags), Description: `The freezer-migrate command checks your database for receipts in a legacy format and updates those. @@ -276,7 +274,7 @@ func inspect(ctx *cli.Context) error { start []byte ) if ctx.NArg() > 2 { - return fmt.Errorf("Max 2 arguments: %v", ctx.Command.ArgsUsage) + return fmt.Errorf("max 2 arguments: %v", ctx.Command.ArgsUsage) } if ctx.NArg() >= 1 { if d, err := hexutil.Decode(ctx.Args().Get(0)); err != nil { @@ -339,9 +337,9 @@ func checkStateContent(ctx *cli.Context) error { hasher.Read(got) if !bytes.Equal(k, got) { errs++ - fmt.Printf("Error at 0x%x\n", k) - fmt.Printf(" Hash: 0x%x\n", got) - fmt.Printf(" Data: 0x%x\n", v) + fmt.Printf("Error at %#x\n", k) + fmt.Printf(" Hash: %#x\n", got) + fmt.Printf(" Data: %#x\n", v) } if time.Since(lastLog) > 8*time.Second { log.Info("Iterating the database", "at", fmt.Sprintf("%#x", k), "elapsed", common.PrettyDuration(time.Since(startTime))) @@ -418,7 +416,7 @@ func dbGet(ctx *cli.Context) error { data, err := db.Get(key) if err != nil { - log.Info("Get operation failed", "key", fmt.Sprintf("0x%#x", key), "error", err) + log.Info("Get operation failed", "key", fmt.Sprintf("%#x", key), "error", err) return err } fmt.Printf("key %#x: %#x\n", key, data) @@ -446,7 +444,7 @@ func dbDelete(ctx *cli.Context) error { fmt.Printf("Previous value: %#x\n", data) } if err = db.Delete(key); err != nil { - log.Info("Delete operation returned an error", "key", fmt.Sprintf("0x%#x", key), "error", err) + log.Info("Delete operation returned an error", "key", fmt.Sprintf("%#x", key), "error", err) return err } return nil @@ -488,7 +486,7 @@ func dbPut(ctx *cli.Context) error { // dbDumpTrie shows the key-value slots of a given storage trie func dbDumpTrie(ctx *cli.Context) error { - if ctx.NArg() < 1 { + if ctx.NArg() < 3 { return fmt.Errorf("required arguments: %v", ctx.Command.ArgsUsage) } stack, _ := makeConfigNode(ctx) @@ -496,30 +494,41 @@ func dbDumpTrie(ctx *cli.Context) error { db := utils.MakeChainDatabase(ctx, stack, true) defer db.Close() + var ( - root []byte - start []byte - max = int64(-1) - err error + state []byte + storage []byte + account []byte + start []byte + max = int64(-1) + err error ) - if root, err = hexutil.Decode(ctx.Args().Get(0)); err != nil { - log.Info("Could not decode the root", "error", err) + if state, err = hexutil.Decode(ctx.Args().Get(0)); err != nil { + log.Info("Could not decode the state root", "error", err) return err } - stRoot := common.BytesToHash(root) - if ctx.NArg() >= 2 { - if start, err = hexutil.Decode(ctx.Args().Get(1)); err != nil { + if account, err = hexutil.Decode(ctx.Args().Get(1)); err != nil { + log.Info("Could not decode the account hash", "error", err) + return err + } + if storage, err = hexutil.Decode(ctx.Args().Get(2)); err != nil { + log.Info("Could not decode the storage trie root", "error", err) + return err + } + if ctx.NArg() > 3 { + if start, err = hexutil.Decode(ctx.Args().Get(3)); err != nil { log.Info("Could not decode the seek position", "error", err) return err } } - if ctx.NArg() >= 3 { - if max, err = strconv.ParseInt(ctx.Args().Get(2), 10, 64); err != nil { + if ctx.NArg() > 4 { + if max, err = strconv.ParseInt(ctx.Args().Get(4), 10, 64); err != nil { log.Info("Could not decode the max count", "error", err) return err } } - theTrie, err := trie.New(stRoot, trie.NewDatabase(db)) + id := trie.StorageTrieID(common.BytesToHash(state), common.BytesToHash(account), common.BytesToHash(storage)) + theTrie, err := trie.New(id, trie.NewDatabase(db)) if err != nil { return err } @@ -537,43 +546,35 @@ func dbDumpTrie(ctx *cli.Context) error { } func freezerInspect(ctx *cli.Context) error { - var ( - start, end int64 - disableSnappy bool - err error - ) - if ctx.NArg() < 3 { + if ctx.NArg() < 4 { return fmt.Errorf("required arguments: %v", ctx.Command.ArgsUsage) } - kind := ctx.Args().Get(0) - if noSnap, ok := rawdb.FreezerNoSnappy[kind]; !ok { - var options []string - for opt := range rawdb.FreezerNoSnappy { - options = append(options, opt) - } - sort.Strings(options) - return fmt.Errorf("Could read freezer-type '%v'. Available options: %v", kind, options) - } else { - disableSnappy = noSnap - } - if start, err = strconv.ParseInt(ctx.Args().Get(1), 10, 64); err != nil { - log.Info("Could read start-param", "error", err) + var ( + freezer = ctx.Args().Get(0) + table = ctx.Args().Get(1) + ) + start, err := strconv.ParseInt(ctx.Args().Get(2), 10, 64) + if err != nil { + log.Info("Could not read start-param", "err", err) return err } - if end, err = strconv.ParseInt(ctx.Args().Get(2), 10, 64); err != nil { - log.Info("Could read count param", "error", err) + end, err := strconv.ParseInt(ctx.Args().Get(3), 10, 64) + if err != nil { + log.Info("Could not read count param", "err", err) return err } stack, _ := makeConfigNode(ctx) defer stack.Close() - path := filepath.Join(stack.ResolvePath("chaindata"), "ancient") - log.Info("Opening freezer", "location", path, "name", kind) - if f, err := rawdb.NewFreezerTable(path, kind, disableSnappy, true); err != nil { + + db := utils.MakeChainDatabase(ctx, stack, true) + defer db.Close() + + ancient, err := db.AncientDatadir() + if err != nil { + log.Info("Failed to retrieve ancient root", "err", err) return err - } else { - f.DumpIndex(start, end) } - return nil + return rawdb.InspectFreezerTable(ancient, freezer, table, start, end) } func importLDBdata(ctx *cli.Context) error { @@ -718,7 +719,7 @@ func showMetaData(ctx *cli.Context) error { if val == nil { return "" } - return fmt.Sprintf("%d (0x%x)", *val, *val) + return fmt.Sprintf("%d (%#x)", *val, *val) } data := [][]string{ {"databaseVersion", pp(rawdb.ReadDatabaseVersion(db))}, @@ -728,7 +729,7 @@ func showMetaData(ctx *cli.Context) error { if b := rawdb.ReadHeadBlock(db); b != nil { data = append(data, []string{"headBlock.Hash", fmt.Sprintf("%v", b.Hash())}) data = append(data, []string{"headBlock.Root", fmt.Sprintf("%v", b.Root())}) - data = append(data, []string{"headBlock.Number", fmt.Sprintf("%d (0x%x)", b.Number(), b.Number())}) + data = append(data, []string{"headBlock.Number", fmt.Sprintf("%d (%#x)", b.Number(), b.Number())}) } if b := rawdb.ReadSkeletonSyncStatus(db); b != nil { data = append(data, []string{"SkeletonSyncStatus", string(b)}) @@ -736,7 +737,7 @@ func showMetaData(ctx *cli.Context) error { if h := rawdb.ReadHeadHeader(db); h != nil { data = append(data, []string{"headHeader.Hash", fmt.Sprintf("%v", h.Hash())}) data = append(data, []string{"headHeader.Root", fmt.Sprintf("%v", h.Root)}) - data = append(data, []string{"headHeader.Number", fmt.Sprintf("%d (0x%x)", h.Number, h.Number)}) + data = append(data, []string{"headHeader.Number", fmt.Sprintf("%d (%#x)", h.Number, h.Number)}) } data = append(data, [][]string{{"frozen", fmt.Sprintf("%d items", ancients)}, {"lastPivotNumber", pp(rawdb.ReadLastPivotNumber(db))}, @@ -832,11 +833,15 @@ func dbHasLegacyReceipts(db ethdb.Database, firstIdx uint64) (bool, uint64, erro } } } - // Is first non-empty receipt legacy? first, err := db.Ancient("receipts", firstIdx) if err != nil { return false, 0, err } + // We looped over all receipts and they were all empty + if bytes.Equal(first, emptyRLPList) { + return false, 0, nil + } + // Is first non-empty receipt legacy? legacy, err = types.IsLegacyStoredReceipts(first) return legacy, firstIdx, err } diff --git a/cmd/geth/genesis_test.go b/cmd/geth/genesis_test.go index c95755f2d9193..7667a85811589 100644 --- a/cmd/geth/genesis_test.go +++ b/cmd/geth/genesis_test.go @@ -83,7 +83,7 @@ func TestCustomGenesis(t *testing.T) { // Query the custom genesis block geth := runGeth(t, "--networkid", "1337", "--syncmode=full", "--cache", "16", - "--datadir", datadir, "--maxpeers", "0", "--port", "0", + "--datadir", datadir, "--maxpeers", "0", "--port", "0", "--authrpc.port", "0", "--nodiscover", "--nat", "none", "--ipcdisable", "--exec", tt.query, "console") geth.ExpectRegexp(tt.result) diff --git a/cmd/geth/les_test.go b/cmd/geth/les_test.go index 73cc23e6674fe..d06c3b91d8a50 100644 --- a/cmd/geth/les_test.go +++ b/cmd/geth/les_test.go @@ -81,41 +81,6 @@ func (g *gethrpc) getNodeInfo() *p2p.NodeInfo { return g.nodeInfo } -func (g *gethrpc) waitSynced() { - // Check if it's synced now - var result interface{} - g.callRPC(&result, "eth_syncing") - syncing, ok := result.(bool) - if ok && !syncing { - g.geth.Logf("%v already synced", g.name) - return - } - - // Actually wait, subscribe to the event - ch := make(chan interface{}) - sub, err := g.rpc.Subscribe(context.Background(), "eth", ch, "syncing") - if err != nil { - g.geth.Fatalf("%v syncing: %v", g.name, err) - } - defer sub.Unsubscribe() - timeout := time.After(4 * time.Second) - select { - case ev := <-ch: - g.geth.Log("'syncing' event", ev) - syncing, ok := ev.(bool) - if ok && !syncing { - break - } - g.geth.Log("Other 'syncing' event", ev) - case err := <-sub.Err(): - g.geth.Fatalf("%v notification: %v", g.name, err) - break - case <-timeout: - g.geth.Fatalf("%v timeout syncing", g.name) - break - } -} - // ipcEndpoint resolves an IPC endpoint based on a configured value, taking into // account the set data folders as well as the designated platform we're currently // running on. @@ -146,7 +111,7 @@ var nextIPC = uint32(0) func startGethWithIpc(t *testing.T, name string, args ...string) *gethrpc { ipcName := fmt.Sprintf("geth-%d.ipc", atomic.AddUint32(&nextIPC, 1)) - args = append([]string{"--networkid=42", "--port=0", "--ipcpath", ipcName}, args...) + args = append([]string{"--networkid=42", "--port=0", "--authrpc.port", "0", "--ipcpath", ipcName}, args...) t.Logf("Starting %v with rpc: %v", name, args) g := &gethrpc{ @@ -179,7 +144,7 @@ func initGeth(t *testing.T) string { func startLightServer(t *testing.T) *gethrpc { datadir := initGeth(t) t.Logf("Importing keys to geth") - runGeth(t, "--datadir", datadir, "--password", "./testdata/password.txt", "account", "import", "./testdata/key.prv", "--lightkdf").WaitExit() + runGeth(t, "account", "import", "--datadir", datadir, "--password", "./testdata/password.txt", "--lightkdf", "./testdata/key.prv").WaitExit() account := "0x02f0d131f1f97aef08aec6e3291b957d9efe7105" server := startGethWithIpc(t, "lightserver", "--allow-insecure-unlock", "--datadir", datadir, "--password", "./testdata/password.txt", "--unlock", account, "--mine", "--light.serve=100", "--light.maxpeers=1", "--nodiscover", "--nat=extip:127.0.0.1", "--verbosity=4") return server diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 1e2770ae808e8..5d54ee41ca2f3 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -44,7 +44,7 @@ import ( _ "github.com/ethereum/go-ethereum/eth/tracers/js" _ "github.com/ethereum/go-ethereum/eth/tracers/native" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) const ( @@ -52,13 +52,8 @@ const ( ) var ( - // Git SHA1 commit hash of the release (set via linker flags) - gitCommit = "" - gitDate = "" - // The app that holds all commands and flags. - app = flags.NewApp(gitCommit, gitDate, "the go-ethereum command line interface") // flags that configure the node - nodeFlags = utils.GroupFlags([]cli.Flag{ + nodeFlags = flags.Merge([]cli.Flag{ utils.IdentityFlag, utils.UnlockedAccountFlag, utils.PasswordFileFlag, @@ -69,8 +64,8 @@ var ( utils.NoUSBFlag, utils.USBFlag, utils.SmartCardDaemonPathFlag, - utils.OverrideArrowGlacierFlag, utils.OverrideTerminalTotalDifficulty, + utils.OverrideTerminalTotalDifficultyPassed, utils.EthashCacheDirFlag, utils.EthashCachesInMemoryFlag, utils.EthashCachesOnDiskFlag, @@ -117,14 +112,15 @@ var ( utils.CacheSnapshotFlag, utils.CacheNoPrefetchFlag, utils.CachePreimagesFlag, + utils.CacheLogSizeFlag, utils.FDLimitFlag, utils.ListenPortFlag, + utils.DiscoveryPortFlag, utils.MaxPeersFlag, utils.MaxPendingPeersFlag, utils.MiningEnabledFlag, utils.MinerThreadsFlag, utils.MinerNotifyFlag, - utils.LegacyMinerGasTargetFlag, utils.MinerGasLimitFlag, utils.MinerGasPriceFlag, utils.MinerEtherbaseFlag, @@ -203,12 +199,14 @@ var ( } ) +var app = flags.NewApp("the go-ethereum command line interface") + func init() { // Initialize the CLI app and start Geth app.Action = geth app.HideVersion = true // we have a command to print the version app.Copyright = "Copyright 2013-2022 The go-ethereum Authors" - app.Commands = []cli.Command{ + app.Commands = []*cli.Command{ // See chaincmd.go: initCommand, importCommand, @@ -239,16 +237,21 @@ func init() { utils.ShowDeprecated, // See snapshot.go snapshotCommand, + // See verkle.go + verkleCommand, } sort.Sort(cli.CommandsByName(app.Commands)) - app.Flags = utils.GroupFlags(nodeFlags, + app.Flags = flags.Merge( + nodeFlags, rpcFlags, consoleFlags, debug.Flags, - metricsFlags) + metricsFlags, + ) app.Before = func(ctx *cli.Context) error { + flags.MigrateGlobalFlags(ctx) return debug.Setup(ctx) } app.After = func(ctx *cli.Context) error { @@ -270,22 +273,22 @@ func main() { func prepare(ctx *cli.Context) { // If we're running a known preset, log it for convenience. switch { - case ctx.GlobalIsSet(utils.RopstenFlag.Name): + case ctx.IsSet(utils.RopstenFlag.Name): log.Info("Starting Geth on Ropsten testnet...") - case ctx.GlobalIsSet(utils.RinkebyFlag.Name): + case ctx.IsSet(utils.RinkebyFlag.Name): log.Info("Starting Geth on Rinkeby testnet...") - case ctx.GlobalIsSet(utils.GoerliFlag.Name): + case ctx.IsSet(utils.GoerliFlag.Name): log.Info("Starting Geth on Görli testnet...") - case ctx.GlobalIsSet(utils.SepoliaFlag.Name): + case ctx.IsSet(utils.SepoliaFlag.Name): log.Info("Starting Geth on Sepolia testnet...") - case ctx.GlobalIsSet(utils.KilnFlag.Name): + case ctx.IsSet(utils.KilnFlag.Name): log.Info("Starting Geth on Kiln testnet...") - case ctx.GlobalIsSet(utils.DeveloperFlag.Name): + case ctx.IsSet(utils.DeveloperFlag.Name): log.Info("Starting Geth in ephemeral dev mode...") log.Warn(`You are running Geth in --dev mode. Please note the following: @@ -303,27 +306,27 @@ func prepare(ctx *cli.Context) { to 0, and discovery is disabled. `) - case !ctx.GlobalIsSet(utils.NetworkIdFlag.Name): + case !ctx.IsSet(utils.NetworkIdFlag.Name): log.Info("Starting Geth on Ethereum mainnet...") } // If we're a full node on mainnet without --cache specified, bump default cache allowance - if ctx.GlobalString(utils.SyncModeFlag.Name) != "light" && !ctx.GlobalIsSet(utils.CacheFlag.Name) && !ctx.GlobalIsSet(utils.NetworkIdFlag.Name) { + if ctx.String(utils.SyncModeFlag.Name) != "light" && !ctx.IsSet(utils.CacheFlag.Name) && !ctx.IsSet(utils.NetworkIdFlag.Name) { // Make sure we're not on any supported preconfigured testnet either - if !ctx.GlobalIsSet(utils.RopstenFlag.Name) && - !ctx.GlobalIsSet(utils.SepoliaFlag.Name) && - !ctx.GlobalIsSet(utils.RinkebyFlag.Name) && - !ctx.GlobalIsSet(utils.GoerliFlag.Name) && - !ctx.GlobalIsSet(utils.KilnFlag.Name) && - !ctx.GlobalIsSet(utils.DeveloperFlag.Name) { + if !ctx.IsSet(utils.RopstenFlag.Name) && + !ctx.IsSet(utils.SepoliaFlag.Name) && + !ctx.IsSet(utils.RinkebyFlag.Name) && + !ctx.IsSet(utils.GoerliFlag.Name) && + !ctx.IsSet(utils.KilnFlag.Name) && + !ctx.IsSet(utils.DeveloperFlag.Name) { // Nope, we're really on mainnet. Bump that cache up! - log.Info("Bumping default cache on mainnet", "provided", ctx.GlobalInt(utils.CacheFlag.Name), "updated", 4096) - ctx.GlobalSet(utils.CacheFlag.Name, strconv.Itoa(4096)) + log.Info("Bumping default cache on mainnet", "provided", ctx.Int(utils.CacheFlag.Name), "updated", 4096) + ctx.Set(utils.CacheFlag.Name, strconv.Itoa(4096)) } } // If we're running a light client on any network, drop the cache to some meaningfully low amount - if ctx.GlobalString(utils.SyncModeFlag.Name) == "light" && !ctx.GlobalIsSet(utils.CacheFlag.Name) { - log.Info("Dropping default light client cache", "provided", ctx.GlobalInt(utils.CacheFlag.Name), "updated", 128) - ctx.GlobalSet(utils.CacheFlag.Name, strconv.Itoa(128)) + if ctx.String(utils.SyncModeFlag.Name) == "light" && !ctx.IsSet(utils.CacheFlag.Name) { + log.Info("Dropping default light client cache", "provided", ctx.Int(utils.CacheFlag.Name), "updated", 128) + ctx.Set(utils.CacheFlag.Name, strconv.Itoa(128)) } // Start metrics export if enabled @@ -337,7 +340,7 @@ func prepare(ctx *cli.Context) { // It creates a default node based on the command line arguments and runs it in // blocking mode, waiting for it to be shut down. func geth(ctx *cli.Context) error { - if args := ctx.Args(); len(args) > 0 { + if args := ctx.Args().Slice(); len(args) > 0 { return fmt.Errorf("invalid command: %q", args[0]) } @@ -408,7 +411,7 @@ func startNode(ctx *cli.Context, stack *node.Node, backend ethapi.Backend, isCon // Spawn a standalone goroutine for status synchronization monitoring, // close the node when synchronization is complete if user required. - if ctx.GlobalBool(utils.ExitWhenSyncedFlag.Name) { + if ctx.Bool(utils.ExitWhenSyncedFlag.Name) { go func() { sub := stack.EventMux().Subscribe(downloader.DoneEvent{}) defer sub.Unsubscribe() @@ -431,9 +434,9 @@ func startNode(ctx *cli.Context, stack *node.Node, backend ethapi.Backend, isCon } // Start auxiliary services if enabled - if ctx.GlobalBool(utils.MiningEnabledFlag.Name) || ctx.GlobalBool(utils.DeveloperFlag.Name) { + if ctx.Bool(utils.MiningEnabledFlag.Name) || ctx.Bool(utils.DeveloperFlag.Name) { // Mining only makes sense if a full Ethereum node is running - if ctx.GlobalString(utils.SyncModeFlag.Name) == "light" { + if ctx.String(utils.SyncModeFlag.Name) == "light" { utils.Fatalf("Light clients do not support mining") } ethBackend, ok := backend.(*eth.EthAPIBackend) @@ -441,10 +444,10 @@ func startNode(ctx *cli.Context, stack *node.Node, backend ethapi.Backend, isCon utils.Fatalf("Ethereum service not running") } // Set the gas price to the limits from the CLI and start mining - gasprice := utils.GlobalBig(ctx, utils.MinerGasPriceFlag.Name) + gasprice := flags.GlobalBig(ctx, utils.MinerGasPriceFlag.Name) ethBackend.TxPool().SetGasPrice(gasprice) // start mining - threads := ctx.GlobalInt(utils.MinerThreadsFlag.Name) + threads := ctx.Int(utils.MinerThreadsFlag.Name) if err := ethBackend.StartMining(threads); err != nil { utils.Fatalf("Failed to start mining: %v", err) } @@ -454,7 +457,7 @@ func startNode(ctx *cli.Context, stack *node.Node, backend ethapi.Backend, isCon // unlockAccounts unlocks any account specifically requested. func unlockAccounts(ctx *cli.Context, stack *node.Node) { var unlocks []string - inputs := strings.Split(ctx.GlobalString(utils.UnlockedAccountFlag.Name), ",") + inputs := strings.Split(ctx.String(utils.UnlockedAccountFlag.Name), ",") for _, input := range inputs { if trimmed := strings.TrimSpace(input); trimmed != "" { unlocks = append(unlocks, trimmed) diff --git a/cmd/geth/misccmd.go b/cmd/geth/misccmd.go index b347d31d97e18..d8a523c632216 100644 --- a/cmd/geth/misccmd.go +++ b/cmd/geth/misccmd.go @@ -25,29 +25,27 @@ import ( "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/internal/version" "github.com/ethereum/go-ethereum/params" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) var ( - VersionCheckUrlFlag = cli.StringFlag{ + VersionCheckUrlFlag = &cli.StringFlag{ Name: "check.url", Usage: "URL to use when checking vulnerabilities", Value: "https://geth.ethereum.org/docs/vulnerabilities/vulnerabilities.json", } - VersionCheckVersionFlag = cli.StringFlag{ + VersionCheckVersionFlag = &cli.StringFlag{ Name: "check.version", Usage: "Version to check", - Value: fmt.Sprintf("Geth/v%v/%v-%v/%v", - params.VersionWithCommit(gitCommit, gitDate), - runtime.GOOS, runtime.GOARCH, runtime.Version()), + Value: version.ClientName(clientIdentifier), } - makecacheCommand = cli.Command{ - Action: utils.MigrateFlags(makecache), + makecacheCommand = &cli.Command{ + Action: makecache, Name: "makecache", Usage: "Generate ethash verification cache (for testing)", ArgsUsage: " ", - Category: "MISCELLANEOUS COMMANDS", Description: ` The makecache command generates an ethash cache in . @@ -55,12 +53,11 @@ This command exists to support the system testing project. Regular users do not need to execute it. `, } - makedagCommand = cli.Command{ - Action: utils.MigrateFlags(makedag), + makedagCommand = &cli.Command{ + Action: makedag, Name: "makedag", Usage: "Generate ethash mining DAG (for testing)", ArgsUsage: " ", - Category: "MISCELLANEOUS COMMANDS", Description: ` The makedag command generates an ethash DAG in . @@ -68,43 +65,40 @@ This command exists to support the system testing project. Regular users do not need to execute it. `, } - versionCommand = cli.Command{ - Action: utils.MigrateFlags(version), + versionCommand = &cli.Command{ + Action: printVersion, Name: "version", Usage: "Print version numbers", ArgsUsage: " ", - Category: "MISCELLANEOUS COMMANDS", Description: ` The output of this command is supposed to be machine-readable. `, } - versionCheckCommand = cli.Command{ - Action: utils.MigrateFlags(versionCheck), + versionCheckCommand = &cli.Command{ + Action: versionCheck, Flags: []cli.Flag{ VersionCheckUrlFlag, VersionCheckVersionFlag, }, Name: "version-check", - Usage: "Checks (online) whether the current version suffers from any known security vulnerabilities", + Usage: "Checks (online) for known Geth security vulnerabilities", ArgsUsage: "", - Category: "MISCELLANEOUS COMMANDS", Description: ` The version-check command fetches vulnerability-information from https://geth.ethereum.org/docs/vulnerabilities/vulnerabilities.json, and displays information about any security vulnerabilities that affect the currently executing version. `, } - licenseCommand = cli.Command{ - Action: utils.MigrateFlags(license), + licenseCommand = &cli.Command{ + Action: license, Name: "license", Usage: "Display license information", ArgsUsage: " ", - Category: "MISCELLANEOUS COMMANDS", } ) // makecache generates an ethash verification cache into the provided folder. func makecache(ctx *cli.Context) error { - args := ctx.Args() + args := ctx.Args().Slice() if len(args) != 2 { utils.Fatalf(`Usage: geth makecache `) } @@ -119,7 +113,7 @@ func makecache(ctx *cli.Context) error { // makedag generates an ethash mining DAG into the provided folder. func makedag(ctx *cli.Context) error { - args := ctx.Args() + args := ctx.Args().Slice() if len(args) != 2 { utils.Fatalf(`Usage: geth makedag `) } @@ -132,14 +126,16 @@ func makedag(ctx *cli.Context) error { return nil } -func version(ctx *cli.Context) error { +func printVersion(ctx *cli.Context) error { + git, _ := version.VCS() + fmt.Println(strings.Title(clientIdentifier)) fmt.Println("Version:", params.VersionWithMeta) - if gitCommit != "" { - fmt.Println("Git Commit:", gitCommit) + if git.Commit != "" { + fmt.Println("Git Commit:", git.Commit) } - if gitDate != "" { - fmt.Println("Git Commit Date:", gitDate) + if git.Date != "" { + fmt.Println("Git Commit Date:", git.Date) } fmt.Println("Architecture:", runtime.GOARCH) fmt.Println("Go Version:", runtime.Version()) diff --git a/cmd/geth/snapshot.go b/cmd/geth/snapshot.go index 9ffc5918cc749..a556f36a416a7 100644 --- a/cmd/geth/snapshot.go +++ b/cmd/geth/snapshot.go @@ -31,10 +31,11 @@ import ( "github.com/ethereum/go-ethereum/core/state/snapshot" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/internal/flags" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" - cli "gopkg.in/urfave/cli.v1" + cli "github.com/urfave/cli/v2" ) var ( @@ -46,19 +47,17 @@ var ( ) var ( - snapshotCommand = cli.Command{ + snapshotCommand = &cli.Command{ Name: "snapshot", Usage: "A set of commands based on the snapshot", - Category: "MISCELLANEOUS COMMANDS", Description: "", - Subcommands: []cli.Command{ + Subcommands: []*cli.Command{ { Name: "prune-state", Usage: "Prune stale ethereum state data based on the snapshot", ArgsUsage: "", - Action: utils.MigrateFlags(pruneState), - Category: "MISCELLANEOUS COMMANDS", - Flags: utils.GroupFlags([]cli.Flag{ + Action: pruneState, + Flags: flags.Merge([]cli.Flag{ utils.CacheTrieJournalFlag, utils.BloomFilterSizeFlag, }, utils.NetworkFlags, utils.DatabasePathFlags), @@ -81,9 +80,8 @@ the trie clean cache with default directory will be deleted. Name: "verify-state", Usage: "Recalculate state hash based on the snapshot for verification", ArgsUsage: "", - Action: utils.MigrateFlags(verifyState), - Category: "MISCELLANEOUS COMMANDS", - Flags: utils.GroupFlags(utils.NetworkFlags, utils.DatabasePathFlags), + Action: verifyState, + Flags: flags.Merge(utils.NetworkFlags, utils.DatabasePathFlags), Description: ` geth snapshot verify-state will traverse the whole accounts and storages set based on the specified @@ -95,21 +93,30 @@ In other words, this command does the snapshot to trie conversion. Name: "check-dangling-storage", Usage: "Check that there is no 'dangling' snap storage", ArgsUsage: "", - Action: utils.MigrateFlags(checkDanglingStorage), - Category: "MISCELLANEOUS COMMANDS", - Flags: utils.GroupFlags(utils.NetworkFlags, utils.DatabasePathFlags), + Action: checkDanglingStorage, + Flags: flags.Merge(utils.NetworkFlags, utils.DatabasePathFlags), Description: ` geth snapshot check-dangling-storage traverses the snap storage data, and verifies that all snapshot storage data has a corresponding account. +`, + }, + { + Name: "inspect-account", + Usage: "Check all snapshot layers for the a specific account", + ArgsUsage: "
", + Action: checkAccount, + Flags: flags.Merge(utils.NetworkFlags, utils.DatabasePathFlags), + Description: ` +geth snapshot inspect-account
checks all snapshot layers and prints out +information about the specified address. `, }, { Name: "traverse-state", Usage: "Traverse the state with given root hash and perform quick verification", ArgsUsage: "", - Action: utils.MigrateFlags(traverseState), - Category: "MISCELLANEOUS COMMANDS", - Flags: utils.GroupFlags(utils.NetworkFlags, utils.DatabasePathFlags), + Action: traverseState, + Flags: flags.Merge(utils.NetworkFlags, utils.DatabasePathFlags), Description: ` geth snapshot traverse-state will traverse the whole state from the given state root and will abort if any @@ -123,9 +130,8 @@ It's also usable without snapshot enabled. Name: "traverse-rawstate", Usage: "Traverse the state with given root hash and perform detailed verification", ArgsUsage: "", - Action: utils.MigrateFlags(traverseRawState), - Category: "MISCELLANEOUS COMMANDS", - Flags: utils.GroupFlags(utils.NetworkFlags, utils.DatabasePathFlags), + Action: traverseRawState, + Flags: flags.Merge(utils.NetworkFlags, utils.DatabasePathFlags), Description: ` geth snapshot traverse-rawstate will traverse the whole state from the given root and will abort if any referenced @@ -140,9 +146,8 @@ It's also usable without snapshot enabled. Name: "dump", Usage: "Dump a specific block from storage (same as 'geth dump' but using snapshots)", ArgsUsage: "[? | ]", - Action: utils.MigrateFlags(dumpState), - Category: "MISCELLANEOUS COMMANDS", - Flags: utils.GroupFlags([]cli.Flag{ + Action: dumpState, + Flags: flags.Merge([]cli.Flag{ utils.ExcludeCodeFlag, utils.ExcludeStorageFlag, utils.StartKeyFlag, @@ -165,7 +170,14 @@ func pruneState(ctx *cli.Context) error { defer stack.Close() chaindb := utils.MakeChainDatabase(ctx, stack, false) - pruner, err := pruner.NewPruner(chaindb, stack.ResolvePath(""), stack.ResolvePath(config.Eth.TrieCleanCacheJournal), ctx.GlobalUint64(utils.BloomFilterSizeFlag.Name)) + defer chaindb.Close() + + prunerconfig := pruner.Config{ + Datadir: stack.ResolvePath(""), + Cachedir: stack.ResolvePath(config.Eth.TrieCleanCacheJournal), + BloomSize: ctx.Uint64(utils.BloomFilterSizeFlag.Name), + } + pruner, err := pruner.NewPruner(chaindb, prunerconfig) if err != nil { log.Error("Failed to open snapshot tree", "err", err) return err @@ -176,7 +188,7 @@ func pruneState(ctx *cli.Context) error { } var targetRoot common.Hash if ctx.NArg() == 1 { - targetRoot, err = parseRoot(ctx.Args()[0]) + targetRoot, err = parseRoot(ctx.Args().First()) if err != nil { log.Error("Failed to resolve state root", "err", err) return err @@ -194,12 +206,20 @@ func verifyState(ctx *cli.Context) error { defer stack.Close() chaindb := utils.MakeChainDatabase(ctx, stack, true) + defer chaindb.Close() + headBlock := rawdb.ReadHeadBlock(chaindb) if headBlock == nil { log.Error("Failed to load head block") return errors.New("no head block") } - snaptree, err := snapshot.New(chaindb, trie.NewDatabase(chaindb), 256, headBlock.Root(), false, false, false) + snapconfig := snapshot.Config{ + CacheSize: 256, + Recovery: false, + NoBuild: true, + AsyncBuild: false, + } + snaptree, err := snapshot.New(snapconfig, chaindb, trie.NewDatabase(chaindb), headBlock.Root()) if err != nil { log.Error("Failed to open snapshot tree", "err", err) return err @@ -210,7 +230,7 @@ func verifyState(ctx *cli.Context) error { } var root = headBlock.Root() if ctx.NArg() == 1 { - root, err = parseRoot(ctx.Args()[0]) + root, err = parseRoot(ctx.Args().First()) if err != nil { log.Error("Failed to resolve state root", "err", err) return err @@ -255,7 +275,7 @@ func traverseState(ctx *cli.Context) error { err error ) if ctx.NArg() == 1 { - root, err = parseRoot(ctx.Args()[0]) + root, err = parseRoot(ctx.Args().First()) if err != nil { log.Error("Failed to resolve state root", "err", err) return err @@ -266,7 +286,7 @@ func traverseState(ctx *cli.Context) error { log.Info("Start traversing the state", "root", root, "number", headBlock.NumberU64()) } triedb := trie.NewDatabase(chaindb) - t, err := trie.NewSecure(root, triedb) + t, err := trie.NewStateTrie(trie.StateTrieID(root), triedb) if err != nil { log.Error("Failed to open trie", "root", root, "err", err) return err @@ -287,7 +307,8 @@ func traverseState(ctx *cli.Context) error { return err } if acc.Root != emptyRoot { - storageTrie, err := trie.NewSecure(acc.Root, triedb) + id := trie.StorageTrieID(root, common.BytesToHash(accIter.Key), acc.Root) + storageTrie, err := trie.NewStateTrie(id, triedb) if err != nil { log.Error("Failed to open storage trie", "root", acc.Root, "err", err) return err @@ -344,7 +365,7 @@ func traverseRawState(ctx *cli.Context) error { err error ) if ctx.NArg() == 1 { - root, err = parseRoot(ctx.Args()[0]) + root, err = parseRoot(ctx.Args().First()) if err != nil { log.Error("Failed to resolve state root", "err", err) return err @@ -355,7 +376,7 @@ func traverseRawState(ctx *cli.Context) error { log.Info("Start traversing the state", "root", root, "number", headBlock.NumberU64()) } triedb := trie.NewDatabase(chaindb) - t, err := trie.NewSecure(root, triedb) + t, err := trie.NewStateTrie(trie.StateTrieID(root), triedb) if err != nil { log.Error("Failed to open trie", "root", root, "err", err) return err @@ -401,7 +422,8 @@ func traverseRawState(ctx *cli.Context) error { return errors.New("invalid account") } if acc.Root != emptyRoot { - storageTrie, err := trie.NewSecure(acc.Root, triedb) + id := trie.StorageTrieID(root, common.BytesToHash(accIter.LeafKey()), acc.Root) + storageTrie, err := trie.NewStateTrie(id, triedb) if err != nil { log.Error("Failed to open storage trie", "root", acc.Root, "err", err) return errors.New("missing storage trie") @@ -474,7 +496,13 @@ func dumpState(ctx *cli.Context) error { if err != nil { return err } - snaptree, err := snapshot.New(db, trie.NewDatabase(db), 256, root, false, false, false) + snapConfig := snapshot.Config{ + CacheSize: 256, + Recovery: false, + NoBuild: true, + AsyncBuild: false, + } + snaptree, err := snapshot.New(snapConfig, db, trie.NewDatabase(db), root) if err != nil { return err } @@ -535,3 +563,35 @@ func dumpState(ctx *cli.Context) error { "elapsed", common.PrettyDuration(time.Since(start))) return nil } + +// checkAccount iterates the snap data layers, and looks up the given account +// across all layers. +func checkAccount(ctx *cli.Context) error { + if ctx.NArg() != 1 { + return errors.New("need arg") + } + var ( + hash common.Hash + addr common.Address + ) + switch arg := ctx.Args().First(); len(arg) { + case 40, 42: + addr = common.HexToAddress(arg) + hash = crypto.Keccak256Hash(addr.Bytes()) + case 64, 66: + hash = common.HexToHash(arg) + default: + return errors.New("malformed address or hash") + } + stack, _ := makeConfigNode(ctx) + defer stack.Close() + chaindb := utils.MakeChainDatabase(ctx, stack, true) + defer chaindb.Close() + start := time.Now() + log.Info("Checking difflayer journal", "address", addr, "hash", hash) + if err := snapshot.CheckJournalAccount(chaindb, hash); err != nil { + return err + } + log.Info("Checked the snapshot journalled storage", "time", common.PrettyDuration(time.Since(start))) + return nil +} diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go deleted file mode 100644 index 56a3d053d6408..0000000000000 --- a/cmd/geth/usage.go +++ /dev/null @@ -1,301 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of go-ethereum. -// -// go-ethereum is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// go-ethereum is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with go-ethereum. If not, see . - -// Contains the geth command usage template and generator. - -package main - -import ( - "io" - "sort" - - "github.com/ethereum/go-ethereum/cmd/utils" - "github.com/ethereum/go-ethereum/internal/debug" - "github.com/ethereum/go-ethereum/internal/flags" - "gopkg.in/urfave/cli.v1" -) - -// AppHelpFlagGroups is the application flags, grouped by functionality. -var AppHelpFlagGroups = []flags.FlagGroup{ - { - Name: "ETHEREUM", - Flags: utils.GroupFlags([]cli.Flag{ - configFileFlag, - utils.MinFreeDiskSpaceFlag, - utils.KeyStoreDirFlag, - utils.USBFlag, - utils.SmartCardDaemonPathFlag, - utils.NetworkIdFlag, - utils.SyncModeFlag, - utils.ExitWhenSyncedFlag, - utils.GCModeFlag, - utils.TxLookupLimitFlag, - utils.EthStatsURLFlag, - utils.IdentityFlag, - utils.LightKDFFlag, - utils.EthRequiredBlocksFlag, - }, utils.NetworkFlags, utils.DatabasePathFlags), - }, - { - Name: "LIGHT CLIENT", - Flags: []cli.Flag{ - utils.LightServeFlag, - utils.LightIngressFlag, - utils.LightEgressFlag, - utils.LightMaxPeersFlag, - utils.UltraLightServersFlag, - utils.UltraLightFractionFlag, - utils.UltraLightOnlyAnnounceFlag, - utils.LightNoPruneFlag, - utils.LightNoSyncServeFlag, - }, - }, - { - Name: "DEVELOPER CHAIN", - Flags: []cli.Flag{ - utils.DeveloperFlag, - utils.DeveloperPeriodFlag, - utils.DeveloperGasLimitFlag, - }, - }, - { - Name: "ETHASH", - Flags: []cli.Flag{ - utils.EthashCacheDirFlag, - utils.EthashCachesInMemoryFlag, - utils.EthashCachesOnDiskFlag, - utils.EthashCachesLockMmapFlag, - utils.EthashDatasetDirFlag, - utils.EthashDatasetsInMemoryFlag, - utils.EthashDatasetsOnDiskFlag, - utils.EthashDatasetsLockMmapFlag, - }, - }, - { - Name: "TRANSACTION POOL", - Flags: []cli.Flag{ - utils.TxPoolLocalsFlag, - utils.TxPoolNoLocalsFlag, - utils.TxPoolJournalFlag, - utils.TxPoolRejournalFlag, - utils.TxPoolPriceLimitFlag, - utils.TxPoolPriceBumpFlag, - utils.TxPoolAccountSlotsFlag, - utils.TxPoolGlobalSlotsFlag, - utils.TxPoolAccountQueueFlag, - utils.TxPoolGlobalQueueFlag, - utils.TxPoolLifetimeFlag, - }, - }, - { - Name: "PERFORMANCE TUNING", - Flags: []cli.Flag{ - utils.CacheFlag, - utils.CacheDatabaseFlag, - utils.CacheTrieFlag, - utils.CacheTrieJournalFlag, - utils.CacheTrieRejournalFlag, - utils.CacheGCFlag, - utils.CacheSnapshotFlag, - utils.CacheNoPrefetchFlag, - utils.CachePreimagesFlag, - utils.FDLimitFlag, - }, - }, - { - Name: "ACCOUNT", - Flags: []cli.Flag{ - utils.UnlockedAccountFlag, - utils.PasswordFileFlag, - utils.ExternalSignerFlag, - utils.InsecureUnlockAllowedFlag, - }, - }, - { - Name: "API AND CONSOLE", - Flags: []cli.Flag{ - utils.IPCDisabledFlag, - utils.IPCPathFlag, - utils.HTTPEnabledFlag, - utils.HTTPListenAddrFlag, - utils.HTTPPortFlag, - utils.HTTPApiFlag, - utils.HTTPPathPrefixFlag, - utils.HTTPCORSDomainFlag, - utils.HTTPVirtualHostsFlag, - utils.WSEnabledFlag, - utils.WSListenAddrFlag, - utils.WSPortFlag, - utils.WSApiFlag, - utils.WSPathPrefixFlag, - utils.WSAllowedOriginsFlag, - utils.JWTSecretFlag, - utils.AuthListenFlag, - utils.AuthPortFlag, - utils.AuthVirtualHostsFlag, - utils.GraphQLEnabledFlag, - utils.GraphQLCORSDomainFlag, - utils.GraphQLVirtualHostsFlag, - utils.RPCGlobalGasCapFlag, - utils.RPCGlobalEVMTimeoutFlag, - utils.RPCGlobalTxFeeCapFlag, - utils.AllowUnprotectedTxs, - utils.JSpathFlag, - utils.ExecFlag, - utils.PreloadJSFlag, - }, - }, - { - Name: "NETWORKING", - Flags: []cli.Flag{ - utils.BootnodesFlag, - utils.DNSDiscoveryFlag, - utils.ListenPortFlag, - utils.MaxPeersFlag, - utils.MaxPendingPeersFlag, - utils.NATFlag, - utils.NoDiscoverFlag, - utils.DiscoveryV5Flag, - utils.NetrestrictFlag, - utils.NodeKeyFileFlag, - utils.NodeKeyHexFlag, - }, - }, - { - Name: "MINER", - Flags: []cli.Flag{ - utils.MiningEnabledFlag, - utils.MinerThreadsFlag, - utils.MinerNotifyFlag, - utils.MinerNotifyFullFlag, - utils.MinerGasPriceFlag, - utils.MinerGasLimitFlag, - utils.MinerEtherbaseFlag, - utils.MinerExtraDataFlag, - utils.MinerRecommitIntervalFlag, - utils.MinerNoVerifyFlag, - }, - }, - { - Name: "GAS PRICE ORACLE", - Flags: []cli.Flag{ - utils.GpoBlocksFlag, - utils.GpoPercentileFlag, - utils.GpoMaxGasPriceFlag, - utils.GpoIgnoreGasPriceFlag, - }, - }, - { - Name: "VIRTUAL MACHINE", - Flags: []cli.Flag{ - utils.VMEnableDebugFlag, - }, - }, - { - Name: "LOGGING AND DEBUGGING", - Flags: append([]cli.Flag{ - utils.FakePoWFlag, - utils.NoCompactionFlag, - }, debug.Flags...), - }, - { - Name: "METRICS AND STATS", - Flags: metricsFlags, - }, - { - Name: "ALIASED (deprecated)", - Flags: []cli.Flag{ - utils.NoUSBFlag, - utils.LegacyWhitelistFlag, - }, - }, - { - Name: "MISC", - Flags: []cli.Flag{ - utils.SnapshotFlag, - utils.BloomFilterSizeFlag, - utils.IgnoreLegacyReceiptsFlag, - cli.HelpFlag, - }, - }, -} - -func init() { - // Override the default app help template - cli.AppHelpTemplate = flags.AppHelpTemplate - - // Override the default app help printer, but only for the global app help - originalHelpPrinter := cli.HelpPrinter - cli.HelpPrinter = func(w io.Writer, tmpl string, data interface{}) { - if tmpl == flags.AppHelpTemplate { - // Iterate over all the flags and add any uncategorized ones - categorized := make(map[string]struct{}) - for _, group := range AppHelpFlagGroups { - for _, flag := range group.Flags { - categorized[flag.String()] = struct{}{} - } - } - deprecated := make(map[string]struct{}) - for _, flag := range utils.DeprecatedFlags { - deprecated[flag.String()] = struct{}{} - } - // Only add uncategorized flags if they are not deprecated - var uncategorized []cli.Flag - for _, flag := range data.(*cli.App).Flags { - if _, ok := categorized[flag.String()]; !ok { - if _, ok := deprecated[flag.String()]; !ok { - uncategorized = append(uncategorized, flag) - } - } - } - if len(uncategorized) > 0 { - // Append all ungategorized options to the misc group - miscs := len(AppHelpFlagGroups[len(AppHelpFlagGroups)-1].Flags) - AppHelpFlagGroups[len(AppHelpFlagGroups)-1].Flags = append(AppHelpFlagGroups[len(AppHelpFlagGroups)-1].Flags, uncategorized...) - - // Make sure they are removed afterwards - defer func() { - AppHelpFlagGroups[len(AppHelpFlagGroups)-1].Flags = AppHelpFlagGroups[len(AppHelpFlagGroups)-1].Flags[:miscs] - }() - } - // Render out custom usage screen - originalHelpPrinter(w, tmpl, flags.HelpData{App: data, FlagGroups: AppHelpFlagGroups}) - } else if tmpl == flags.CommandHelpTemplate { - // Iterate over all command specific flags and categorize them - categorized := make(map[string][]cli.Flag) - for _, flag := range data.(cli.Command).Flags { - if _, ok := categorized[flag.String()]; !ok { - categorized[flags.FlagCategory(flag, AppHelpFlagGroups)] = append(categorized[flags.FlagCategory(flag, AppHelpFlagGroups)], flag) - } - } - - // sort to get a stable ordering - sorted := make([]flags.FlagGroup, 0, len(categorized)) - for cat, flgs := range categorized { - sorted = append(sorted, flags.FlagGroup{Name: cat, Flags: flgs}) - } - sort.Sort(flags.ByCategory(sorted)) - - // add sorted array to data and render with default printer - originalHelpPrinter(w, tmpl, map[string]interface{}{ - "cmd": data, - "categorizedFlags": sorted, - }) - } else { - originalHelpPrinter(w, tmpl, data) - } - } -} diff --git a/cmd/geth/verkle.go b/cmd/geth/verkle.go new file mode 100644 index 0000000000000..f85ec37ea924f --- /dev/null +++ b/cmd/geth/verkle.go @@ -0,0 +1,213 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +import ( + "bytes" + "encoding/hex" + "errors" + "fmt" + "os" + + "github.com/ethereum/go-ethereum/cmd/utils" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/internal/flags" + "github.com/ethereum/go-ethereum/log" + "github.com/gballet/go-verkle" + cli "github.com/urfave/cli/v2" +) + +var ( + zero [32]byte + + verkleCommand = &cli.Command{ + Name: "verkle", + Usage: "A set of experimental verkle tree management commands", + Category: "MISCELLANEOUS COMMANDS", + Description: "", + Subcommands: []*cli.Command{ + { + Name: "verify", + Usage: "verify the conversion of a MPT into a verkle tree", + ArgsUsage: "", + Action: verifyVerkle, + Flags: flags.Merge(utils.NetworkFlags, utils.DatabasePathFlags), + Description: ` +geth verkle verify +This command takes a root commitment and attempts to rebuild the tree. + `, + }, + { + Name: "dump", + Usage: "Dump a verkle tree to a DOT file", + ArgsUsage: " [ ...]", + Action: expandVerkle, + Flags: flags.Merge(utils.NetworkFlags, utils.DatabasePathFlags), + Description: ` +geth verkle dump [ ...] +This command will produce a dot file representing the tree, rooted at . +in which key1, key2, ... are expanded. + `, + }, + }, + } +) + +// recurse into each child to ensure they can be loaded from the db. The tree isn't rebuilt +// (only its nodes are loaded) so there is no need to flush them, the garbage collector should +// take care of that for us. +func checkChildren(root verkle.VerkleNode, resolver verkle.NodeResolverFn) error { + switch node := root.(type) { + case *verkle.InternalNode: + for i, child := range node.Children() { + childC := child.ComputeCommitment().Bytes() + + childS, err := resolver(childC[:]) + if bytes.Equal(childC[:], zero[:]) { + continue + } + if err != nil { + return fmt.Errorf("could not find child %x in db: %w", childC, err) + } + // depth is set to 0, the tree isn't rebuilt so it's not a problem + childN, err := verkle.ParseNode(childS, 0, childC[:]) + if err != nil { + return fmt.Errorf("decode error child %x in db: %w", child.ComputeCommitment().Bytes(), err) + } + if err := checkChildren(childN, resolver); err != nil { + return fmt.Errorf("%x%w", i, err) // write the path to the erroring node + } + } + case *verkle.LeafNode: + // sanity check: ensure at least one value is non-zero + + for i := 0; i < verkle.NodeWidth; i++ { + if len(node.Value(i)) != 0 { + return nil + } + } + return fmt.Errorf("Both balance and nonce are 0") + case verkle.Empty: + // nothing to do + default: + return fmt.Errorf("unsupported type encountered %v", root) + } + + return nil +} + +func verifyVerkle(ctx *cli.Context) error { + stack, _ := makeConfigNode(ctx) + defer stack.Close() + + chaindb := utils.MakeChainDatabase(ctx, stack, true) + headBlock := rawdb.ReadHeadBlock(chaindb) + if headBlock == nil { + log.Error("Failed to load head block") + return errors.New("no head block") + } + if ctx.NArg() > 1 { + log.Error("Too many arguments given") + return errors.New("too many arguments") + } + var ( + rootC common.Hash + err error + ) + if ctx.NArg() == 1 { + rootC, err = parseRoot(ctx.Args().First()) + if err != nil { + log.Error("Failed to resolve state root", "error", err) + return err + } + log.Info("Rebuilding the tree", "root", rootC) + } else { + rootC = headBlock.Root() + log.Info("Rebuilding the tree", "root", rootC, "number", headBlock.NumberU64()) + } + + serializedRoot, err := chaindb.Get(rootC[:]) + if err != nil { + return err + } + root, err := verkle.ParseNode(serializedRoot, 0, rootC[:]) + if err != nil { + return err + } + + if err := checkChildren(root, chaindb.Get); err != nil { + log.Error("Could not rebuild the tree from the database", "err", err) + return err + } + + log.Info("Tree was rebuilt from the database") + return nil +} + +func expandVerkle(ctx *cli.Context) error { + stack, _ := makeConfigNode(ctx) + defer stack.Close() + + chaindb := utils.MakeChainDatabase(ctx, stack, true) + var ( + rootC common.Hash + keylist [][]byte + err error + ) + if ctx.NArg() >= 2 { + rootC, err = parseRoot(ctx.Args().First()) + if err != nil { + log.Error("Failed to resolve state root", "error", err) + return err + } + keylist = make([][]byte, 0, ctx.Args().Len()-1) + args := ctx.Args().Slice() + for i := range args[1:] { + key, err := hex.DecodeString(args[i+1]) + log.Info("decoded key", "arg", args[i+1], "key", key) + if err != nil { + return fmt.Errorf("error decoding key #%d: %w", i+1, err) + } + keylist = append(keylist, key) + } + log.Info("Rebuilding the tree", "root", rootC) + } else { + return fmt.Errorf("usage: %s root key1 [key 2...]", ctx.App.Name) + } + + serializedRoot, err := chaindb.Get(rootC[:]) + if err != nil { + return err + } + root, err := verkle.ParseNode(serializedRoot, 0, rootC[:]) + if err != nil { + return err + } + + for i, key := range keylist { + log.Info("Reading key", "index", i, "key", keylist[0]) + root.Get(key, chaindb.Get) + } + + if err := os.WriteFile("dump.dot", []byte(verkle.ToDot(root)), 0600); err != nil { + log.Error("Failed to dump file", "err", err) + } else { + log.Info("Tree was dumped to file", "file", "dump.dot") + } + return nil +} diff --git a/cmd/geth/version_check.go b/cmd/geth/version_check.go index 6eaedf373437e..237556788eb9f 100644 --- a/cmd/geth/version_check.go +++ b/cmd/geth/version_check.go @@ -28,7 +28,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/jedisct1/go-minisign" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) var gethPubKeys []string = []string{ diff --git a/cmd/geth/version_check_test.go b/cmd/geth/version_check_test.go index b841ace5b2f9a..bd4d820a79013 100644 --- a/cmd/geth/version_check_test.go +++ b/cmd/geth/version_check_test.go @@ -118,7 +118,6 @@ func TestMatching(t *testing.T) { version, vuln.Introduced, vuln.Fixed, vuln.Name, vulnIntro, current, vulnFixed) } } - } } for major := 1; major < 2; major++ { diff --git a/cmd/p2psim/main.go b/cmd/p2psim/main.go index eaa457200a431..a3546d405b5f8 100644 --- a/cmd/p2psim/main.go +++ b/cmd/p2psim/main.go @@ -19,21 +19,20 @@ // Here is an example of creating a 2 node network with the first node // connected to the second: // -// $ p2psim node create -// Created node01 +// $ p2psim node create +// Created node01 // -// $ p2psim node start node01 -// Started node01 +// $ p2psim node start node01 +// Started node01 // -// $ p2psim node create -// Created node02 +// $ p2psim node create +// Created node02 // -// $ p2psim node start node02 -// Started node02 -// -// $ p2psim node connect node01 node02 -// Connected node01 to node02 +// $ p2psim node start node02 +// Started node02 // +// $ p2psim node connect node01 node02 +// Connected node01 to node02 package main import ( @@ -46,71 +45,71 @@ import ( "text/tabwriter" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/internal/flags" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/p2p/simulations" "github.com/ethereum/go-ethereum/p2p/simulations/adapters" "github.com/ethereum/go-ethereum/rpc" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) var client *simulations.Client var ( // global command flags - apiFlag = cli.StringFlag{ - Name: "api", - Value: "http://localhost:8888", - Usage: "simulation API URL", - EnvVar: "P2PSIM_API_URL", + apiFlag = &cli.StringFlag{ + Name: "api", + Value: "http://localhost:8888", + Usage: "simulation API URL", + EnvVars: []string{"P2PSIM_API_URL"}, } // events subcommand flags - currentFlag = cli.BoolFlag{ + currentFlag = &cli.BoolFlag{ Name: "current", Usage: "get existing nodes and conns first", } - filterFlag = cli.StringFlag{ + filterFlag = &cli.StringFlag{ Name: "filter", Value: "", Usage: "message filter", } // node create subcommand flags - nameFlag = cli.StringFlag{ + nameFlag = &cli.StringFlag{ Name: "name", Value: "", Usage: "node name", } - servicesFlag = cli.StringFlag{ + servicesFlag = &cli.StringFlag{ Name: "services", Value: "", Usage: "node services (comma separated)", } - keyFlag = cli.StringFlag{ + keyFlag = &cli.StringFlag{ Name: "key", Value: "", Usage: "node private key (hex encoded)", } // node rpc subcommand flags - subscribeFlag = cli.BoolFlag{ + subscribeFlag = &cli.BoolFlag{ Name: "subscribe", Usage: "method is a subscription", } ) func main() { - app := cli.NewApp() - app.Usage = "devp2p simulation command-line client" + app := flags.NewApp("devp2p simulation command-line client") app.Flags = []cli.Flag{ apiFlag, } app.Before = func(ctx *cli.Context) error { - client = simulations.NewClient(ctx.GlobalString(apiFlag.Name)) + client = simulations.NewClient(ctx.String(apiFlag.Name)) return nil } - app.Commands = []cli.Command{ + app.Commands = []*cli.Command{ { Name: "show", Usage: "show network information", @@ -139,7 +138,7 @@ func main() { Name: "node", Usage: "manage simulation nodes", Action: listNodes, - Subcommands: []cli.Command{ + Subcommands: []*cli.Command{ { Name: "list", Usage: "list nodes", @@ -204,7 +203,7 @@ func main() { } func showNetwork(ctx *cli.Context) error { - if len(ctx.Args()) != 0 { + if ctx.NArg() != 0 { return cli.ShowCommandHelp(ctx, ctx.Command.Name) } network, err := client.GetNetwork() @@ -219,7 +218,7 @@ func showNetwork(ctx *cli.Context) error { } func streamNetwork(ctx *cli.Context) error { - if len(ctx.Args()) != 0 { + if ctx.NArg() != 0 { return cli.ShowCommandHelp(ctx, ctx.Command.Name) } events := make(chan *simulations.Event) @@ -245,7 +244,7 @@ func streamNetwork(ctx *cli.Context) error { } func createSnapshot(ctx *cli.Context) error { - if len(ctx.Args()) != 0 { + if ctx.NArg() != 0 { return cli.ShowCommandHelp(ctx, ctx.Command.Name) } snap, err := client.CreateSnapshot() @@ -256,7 +255,7 @@ func createSnapshot(ctx *cli.Context) error { } func loadSnapshot(ctx *cli.Context) error { - if len(ctx.Args()) != 0 { + if ctx.NArg() != 0 { return cli.ShowCommandHelp(ctx, ctx.Command.Name) } snap := &simulations.Snapshot{} @@ -267,7 +266,7 @@ func loadSnapshot(ctx *cli.Context) error { } func listNodes(ctx *cli.Context) error { - if len(ctx.Args()) != 0 { + if ctx.NArg() != 0 { return cli.ShowCommandHelp(ctx, ctx.Command.Name) } nodes, err := client.GetNodes() @@ -292,7 +291,7 @@ func protocolList(node *p2p.NodeInfo) []string { } func createNode(ctx *cli.Context) error { - if len(ctx.Args()) != 0 { + if ctx.NArg() != 0 { return cli.ShowCommandHelp(ctx, ctx.Command.Name) } config := adapters.RandomNodeConfig() @@ -317,11 +316,10 @@ func createNode(ctx *cli.Context) error { } func showNode(ctx *cli.Context) error { - args := ctx.Args() - if len(args) != 1 { + if ctx.NArg() != 1 { return cli.ShowCommandHelp(ctx, ctx.Command.Name) } - nodeName := args[0] + nodeName := ctx.Args().First() node, err := client.GetNode(nodeName) if err != nil { return err @@ -342,11 +340,10 @@ func showNode(ctx *cli.Context) error { } func startNode(ctx *cli.Context) error { - args := ctx.Args() - if len(args) != 1 { + if ctx.NArg() != 1 { return cli.ShowCommandHelp(ctx, ctx.Command.Name) } - nodeName := args[0] + nodeName := ctx.Args().First() if err := client.StartNode(nodeName); err != nil { return err } @@ -355,11 +352,10 @@ func startNode(ctx *cli.Context) error { } func stopNode(ctx *cli.Context) error { - args := ctx.Args() - if len(args) != 1 { + if ctx.NArg() != 1 { return cli.ShowCommandHelp(ctx, ctx.Command.Name) } - nodeName := args[0] + nodeName := ctx.Args().First() if err := client.StopNode(nodeName); err != nil { return err } @@ -368,12 +364,12 @@ func stopNode(ctx *cli.Context) error { } func connectNode(ctx *cli.Context) error { - args := ctx.Args() - if len(args) != 2 { + if ctx.NArg() != 2 { return cli.ShowCommandHelp(ctx, ctx.Command.Name) } - nodeName := args[0] - peerName := args[1] + args := ctx.Args() + nodeName := args.Get(0) + peerName := args.Get(1) if err := client.ConnectNode(nodeName, peerName); err != nil { return err } @@ -383,11 +379,11 @@ func connectNode(ctx *cli.Context) error { func disconnectNode(ctx *cli.Context) error { args := ctx.Args() - if len(args) != 2 { + if args.Len() != 2 { return cli.ShowCommandHelp(ctx, ctx.Command.Name) } - nodeName := args[0] - peerName := args[1] + nodeName := args.Get(0) + peerName := args.Get(1) if err := client.DisconnectNode(nodeName, peerName); err != nil { return err } @@ -397,21 +393,21 @@ func disconnectNode(ctx *cli.Context) error { func rpcNode(ctx *cli.Context) error { args := ctx.Args() - if len(args) < 2 { + if args.Len() < 2 { return cli.ShowCommandHelp(ctx, ctx.Command.Name) } - nodeName := args[0] - method := args[1] + nodeName := args.Get(0) + method := args.Get(1) rpcClient, err := client.RPCClient(context.Background(), nodeName) if err != nil { return err } if ctx.Bool(subscribeFlag.Name) { - return rpcSubscribe(rpcClient, ctx.App.Writer, method, args[3:]...) + return rpcSubscribe(rpcClient, ctx.App.Writer, method, args.Slice()[3:]...) } var result interface{} - params := make([]interface{}, len(args[3:])) - for i, v := range args[3:] { + params := make([]interface{}, len(args.Slice()[3:])) + for i, v := range args.Slice()[3:] { params[i] = v } if err := rpcClient.Call(&result, method, params...); err != nil { diff --git a/cmd/puppeth/genesis.go b/cmd/puppeth/genesis.go deleted file mode 100644 index ef1f977bf09f3..0000000000000 --- a/cmd/puppeth/genesis.go +++ /dev/null @@ -1,626 +0,0 @@ -// Copyright 2017 The go-ethereum Authors -// This file is part of go-ethereum. -// -// go-ethereum is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// go-ethereum is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with go-ethereum. If not, see . - -package main - -import ( - "errors" - "math" - "math/big" - "strings" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/hexutil" - math2 "github.com/ethereum/go-ethereum/common/math" - "github.com/ethereum/go-ethereum/consensus/ethash" - "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/params" -) - -// alethGenesisSpec represents the genesis specification format used by the -// C++ Ethereum implementation. -type alethGenesisSpec struct { - SealEngine string `json:"sealEngine"` - Params struct { - AccountStartNonce math2.HexOrDecimal64 `json:"accountStartNonce"` - MaximumExtraDataSize hexutil.Uint64 `json:"maximumExtraDataSize"` - HomesteadForkBlock *hexutil.Big `json:"homesteadForkBlock,omitempty"` - DaoHardforkBlock math2.HexOrDecimal64 `json:"daoHardforkBlock"` - EIP150ForkBlock *hexutil.Big `json:"EIP150ForkBlock,omitempty"` - EIP158ForkBlock *hexutil.Big `json:"EIP158ForkBlock,omitempty"` - ByzantiumForkBlock *hexutil.Big `json:"byzantiumForkBlock,omitempty"` - ConstantinopleForkBlock *hexutil.Big `json:"constantinopleForkBlock,omitempty"` - ConstantinopleFixForkBlock *hexutil.Big `json:"constantinopleFixForkBlock,omitempty"` - IstanbulForkBlock *hexutil.Big `json:"istanbulForkBlock,omitempty"` - MinGasLimit hexutil.Uint64 `json:"minGasLimit"` - MaxGasLimit hexutil.Uint64 `json:"maxGasLimit"` - TieBreakingGas bool `json:"tieBreakingGas"` - GasLimitBoundDivisor math2.HexOrDecimal64 `json:"gasLimitBoundDivisor"` - MinimumDifficulty *hexutil.Big `json:"minimumDifficulty"` - DifficultyBoundDivisor *math2.HexOrDecimal256 `json:"difficultyBoundDivisor"` - DurationLimit *math2.HexOrDecimal256 `json:"durationLimit"` - BlockReward *hexutil.Big `json:"blockReward"` - NetworkID hexutil.Uint64 `json:"networkID"` - ChainID hexutil.Uint64 `json:"chainID"` - AllowFutureBlocks bool `json:"allowFutureBlocks"` - } `json:"params"` - - Genesis struct { - Nonce types.BlockNonce `json:"nonce"` - Difficulty *hexutil.Big `json:"difficulty"` - MixHash common.Hash `json:"mixHash"` - Author common.Address `json:"author"` - Timestamp hexutil.Uint64 `json:"timestamp"` - ParentHash common.Hash `json:"parentHash"` - ExtraData hexutil.Bytes `json:"extraData"` - GasLimit hexutil.Uint64 `json:"gasLimit"` - } `json:"genesis"` - - Accounts map[common.UnprefixedAddress]*alethGenesisSpecAccount `json:"accounts"` -} - -// alethGenesisSpecAccount is the prefunded genesis account and/or precompiled -// contract definition. -type alethGenesisSpecAccount struct { - Balance *math2.HexOrDecimal256 `json:"balance,omitempty"` - Nonce uint64 `json:"nonce,omitempty"` - Precompiled *alethGenesisSpecBuiltin `json:"precompiled,omitempty"` -} - -// alethGenesisSpecBuiltin is the precompiled contract definition. -type alethGenesisSpecBuiltin struct { - Name string `json:"name,omitempty"` - StartingBlock *hexutil.Big `json:"startingBlock,omitempty"` - Linear *alethGenesisSpecLinearPricing `json:"linear,omitempty"` -} - -type alethGenesisSpecLinearPricing struct { - Base uint64 `json:"base"` - Word uint64 `json:"word"` -} - -// newAlethGenesisSpec converts a go-ethereum genesis block into a Aleth-specific -// chain specification format. -func newAlethGenesisSpec(network string, genesis *core.Genesis) (*alethGenesisSpec, error) { - // Only ethash is currently supported between go-ethereum and aleth - if genesis.Config.Ethash == nil { - return nil, errors.New("unsupported consensus engine") - } - // Reconstruct the chain spec in Aleth format - spec := &alethGenesisSpec{ - SealEngine: "Ethash", - } - // Some defaults - spec.Params.AccountStartNonce = 0 - spec.Params.TieBreakingGas = false - spec.Params.AllowFutureBlocks = false - - // Dao hardfork block is a special one. The fork block is listed as 0 in the - // config but aleth will sync with ETC clients up until the actual dao hard - // fork block. - spec.Params.DaoHardforkBlock = 0 - - if num := genesis.Config.HomesteadBlock; num != nil { - spec.Params.HomesteadForkBlock = (*hexutil.Big)(num) - } - if num := genesis.Config.EIP150Block; num != nil { - spec.Params.EIP150ForkBlock = (*hexutil.Big)(num) - } - if num := genesis.Config.EIP158Block; num != nil { - spec.Params.EIP158ForkBlock = (*hexutil.Big)(num) - } - if num := genesis.Config.ByzantiumBlock; num != nil { - spec.Params.ByzantiumForkBlock = (*hexutil.Big)(num) - } - if num := genesis.Config.ConstantinopleBlock; num != nil { - spec.Params.ConstantinopleForkBlock = (*hexutil.Big)(num) - } - if num := genesis.Config.PetersburgBlock; num != nil { - spec.Params.ConstantinopleFixForkBlock = (*hexutil.Big)(num) - } - if num := genesis.Config.IstanbulBlock; num != nil { - spec.Params.IstanbulForkBlock = (*hexutil.Big)(num) - } - spec.Params.NetworkID = (hexutil.Uint64)(genesis.Config.ChainID.Uint64()) - spec.Params.ChainID = (hexutil.Uint64)(genesis.Config.ChainID.Uint64()) - spec.Params.MaximumExtraDataSize = (hexutil.Uint64)(params.MaximumExtraDataSize) - spec.Params.MinGasLimit = (hexutil.Uint64)(params.MinGasLimit) - spec.Params.MaxGasLimit = (hexutil.Uint64)(math.MaxInt64) - spec.Params.MinimumDifficulty = (*hexutil.Big)(params.MinimumDifficulty) - spec.Params.DifficultyBoundDivisor = (*math2.HexOrDecimal256)(params.DifficultyBoundDivisor) - spec.Params.GasLimitBoundDivisor = (math2.HexOrDecimal64)(params.GasLimitBoundDivisor) - spec.Params.DurationLimit = (*math2.HexOrDecimal256)(params.DurationLimit) - spec.Params.BlockReward = (*hexutil.Big)(ethash.FrontierBlockReward) - - spec.Genesis.Nonce = types.EncodeNonce(genesis.Nonce) - spec.Genesis.MixHash = genesis.Mixhash - spec.Genesis.Difficulty = (*hexutil.Big)(genesis.Difficulty) - spec.Genesis.Author = genesis.Coinbase - spec.Genesis.Timestamp = (hexutil.Uint64)(genesis.Timestamp) - spec.Genesis.ParentHash = genesis.ParentHash - spec.Genesis.ExtraData = genesis.ExtraData - spec.Genesis.GasLimit = (hexutil.Uint64)(genesis.GasLimit) - - for address, account := range genesis.Alloc { - spec.setAccount(address, account) - } - - spec.setPrecompile(1, &alethGenesisSpecBuiltin{Name: "ecrecover", - Linear: &alethGenesisSpecLinearPricing{Base: 3000}}) - spec.setPrecompile(2, &alethGenesisSpecBuiltin{Name: "sha256", - Linear: &alethGenesisSpecLinearPricing{Base: 60, Word: 12}}) - spec.setPrecompile(3, &alethGenesisSpecBuiltin{Name: "ripemd160", - Linear: &alethGenesisSpecLinearPricing{Base: 600, Word: 120}}) - spec.setPrecompile(4, &alethGenesisSpecBuiltin{Name: "identity", - Linear: &alethGenesisSpecLinearPricing{Base: 15, Word: 3}}) - if genesis.Config.ByzantiumBlock != nil { - spec.setPrecompile(5, &alethGenesisSpecBuiltin{Name: "modexp", - StartingBlock: (*hexutil.Big)(genesis.Config.ByzantiumBlock)}) - spec.setPrecompile(6, &alethGenesisSpecBuiltin{Name: "alt_bn128_G1_add", - StartingBlock: (*hexutil.Big)(genesis.Config.ByzantiumBlock), - Linear: &alethGenesisSpecLinearPricing{Base: 500}}) - spec.setPrecompile(7, &alethGenesisSpecBuiltin{Name: "alt_bn128_G1_mul", - StartingBlock: (*hexutil.Big)(genesis.Config.ByzantiumBlock), - Linear: &alethGenesisSpecLinearPricing{Base: 40000}}) - spec.setPrecompile(8, &alethGenesisSpecBuiltin{Name: "alt_bn128_pairing_product", - StartingBlock: (*hexutil.Big)(genesis.Config.ByzantiumBlock)}) - } - if genesis.Config.IstanbulBlock != nil { - if genesis.Config.ByzantiumBlock == nil { - return nil, errors.New("invalid genesis, istanbul fork is enabled while byzantium is not") - } - spec.setPrecompile(6, &alethGenesisSpecBuiltin{ - Name: "alt_bn128_G1_add", - StartingBlock: (*hexutil.Big)(genesis.Config.ByzantiumBlock), - }) // Aleth hardcoded the gas policy - spec.setPrecompile(7, &alethGenesisSpecBuiltin{ - Name: "alt_bn128_G1_mul", - StartingBlock: (*hexutil.Big)(genesis.Config.ByzantiumBlock), - }) // Aleth hardcoded the gas policy - spec.setPrecompile(9, &alethGenesisSpecBuiltin{ - Name: "blake2_compression", - StartingBlock: (*hexutil.Big)(genesis.Config.IstanbulBlock), - }) - } - return spec, nil -} - -func (spec *alethGenesisSpec) setPrecompile(address byte, data *alethGenesisSpecBuiltin) { - if spec.Accounts == nil { - spec.Accounts = make(map[common.UnprefixedAddress]*alethGenesisSpecAccount) - } - addr := common.UnprefixedAddress(common.BytesToAddress([]byte{address})) - if _, exist := spec.Accounts[addr]; !exist { - spec.Accounts[addr] = &alethGenesisSpecAccount{} - } - spec.Accounts[addr].Precompiled = data -} - -func (spec *alethGenesisSpec) setAccount(address common.Address, account core.GenesisAccount) { - if spec.Accounts == nil { - spec.Accounts = make(map[common.UnprefixedAddress]*alethGenesisSpecAccount) - } - - a, exist := spec.Accounts[common.UnprefixedAddress(address)] - if !exist { - a = &alethGenesisSpecAccount{} - spec.Accounts[common.UnprefixedAddress(address)] = a - } - a.Balance = (*math2.HexOrDecimal256)(account.Balance) - a.Nonce = account.Nonce - -} - -// parityChainSpec is the chain specification format used by Parity. -type parityChainSpec struct { - Name string `json:"name"` - Datadir string `json:"dataDir"` - Engine struct { - Ethash struct { - Params struct { - MinimumDifficulty *hexutil.Big `json:"minimumDifficulty"` - DifficultyBoundDivisor *hexutil.Big `json:"difficultyBoundDivisor"` - DurationLimit *hexutil.Big `json:"durationLimit"` - BlockReward map[string]string `json:"blockReward"` - DifficultyBombDelays map[string]string `json:"difficultyBombDelays"` - HomesteadTransition hexutil.Uint64 `json:"homesteadTransition"` - EIP100bTransition hexutil.Uint64 `json:"eip100bTransition"` - } `json:"params"` - } `json:"Ethash"` - } `json:"engine"` - - Params struct { - AccountStartNonce hexutil.Uint64 `json:"accountStartNonce"` - MaximumExtraDataSize hexutil.Uint64 `json:"maximumExtraDataSize"` - MinGasLimit hexutil.Uint64 `json:"minGasLimit"` - GasLimitBoundDivisor math2.HexOrDecimal64 `json:"gasLimitBoundDivisor"` - NetworkID hexutil.Uint64 `json:"networkID"` - ChainID hexutil.Uint64 `json:"chainID"` - MaxCodeSize hexutil.Uint64 `json:"maxCodeSize"` - MaxCodeSizeTransition hexutil.Uint64 `json:"maxCodeSizeTransition"` - EIP98Transition hexutil.Uint64 `json:"eip98Transition"` - EIP150Transition hexutil.Uint64 `json:"eip150Transition"` - EIP160Transition hexutil.Uint64 `json:"eip160Transition"` - EIP161abcTransition hexutil.Uint64 `json:"eip161abcTransition"` - EIP161dTransition hexutil.Uint64 `json:"eip161dTransition"` - EIP155Transition hexutil.Uint64 `json:"eip155Transition"` - EIP140Transition hexutil.Uint64 `json:"eip140Transition"` - EIP211Transition hexutil.Uint64 `json:"eip211Transition"` - EIP214Transition hexutil.Uint64 `json:"eip214Transition"` - EIP658Transition hexutil.Uint64 `json:"eip658Transition"` - EIP145Transition hexutil.Uint64 `json:"eip145Transition"` - EIP1014Transition hexutil.Uint64 `json:"eip1014Transition"` - EIP1052Transition hexutil.Uint64 `json:"eip1052Transition"` - EIP1283Transition hexutil.Uint64 `json:"eip1283Transition"` - EIP1283DisableTransition hexutil.Uint64 `json:"eip1283DisableTransition"` - EIP1283ReenableTransition hexutil.Uint64 `json:"eip1283ReenableTransition"` - EIP1344Transition hexutil.Uint64 `json:"eip1344Transition"` - EIP1884Transition hexutil.Uint64 `json:"eip1884Transition"` - EIP2028Transition hexutil.Uint64 `json:"eip2028Transition"` - } `json:"params"` - - Genesis struct { - Seal struct { - Ethereum struct { - Nonce types.BlockNonce `json:"nonce"` - MixHash hexutil.Bytes `json:"mixHash"` - } `json:"ethereum"` - } `json:"seal"` - - Difficulty *hexutil.Big `json:"difficulty"` - Author common.Address `json:"author"` - Timestamp hexutil.Uint64 `json:"timestamp"` - ParentHash common.Hash `json:"parentHash"` - ExtraData hexutil.Bytes `json:"extraData"` - GasLimit hexutil.Uint64 `json:"gasLimit"` - } `json:"genesis"` - - Nodes []string `json:"nodes"` - Accounts map[common.UnprefixedAddress]*parityChainSpecAccount `json:"accounts"` -} - -// parityChainSpecAccount is the prefunded genesis account and/or precompiled -// contract definition. -type parityChainSpecAccount struct { - Balance math2.HexOrDecimal256 `json:"balance"` - Nonce math2.HexOrDecimal64 `json:"nonce,omitempty"` - Builtin *parityChainSpecBuiltin `json:"builtin,omitempty"` -} - -// parityChainSpecBuiltin is the precompiled contract definition. -type parityChainSpecBuiltin struct { - Name string `json:"name"` // Each builtin should has it own name - Pricing interface{} `json:"pricing"` // Each builtin should has it own price strategy - ActivateAt *hexutil.Big `json:"activate_at,omitempty"` // ActivateAt can't be omitted if empty, default means no fork -} - -// parityChainSpecPricing represents the different pricing models that builtin -// contracts might advertise using. -type parityChainSpecPricing struct { - Linear *parityChainSpecLinearPricing `json:"linear,omitempty"` - ModExp *parityChainSpecModExpPricing `json:"modexp,omitempty"` - - // Before the https://github.com/paritytech/parity-ethereum/pull/11039, - // Parity uses this format to config bn pairing price policy. - AltBnPairing *parityChainSepcAltBnPairingPricing `json:"alt_bn128_pairing,omitempty"` - - // Blake2F is the price per round of Blake2 compression - Blake2F *parityChainSpecBlakePricing `json:"blake2_f,omitempty"` -} - -type parityChainSpecLinearPricing struct { - Base uint64 `json:"base"` - Word uint64 `json:"word"` -} - -type parityChainSpecModExpPricing struct { - Divisor uint64 `json:"divisor"` -} - -// parityChainSpecAltBnConstOperationPricing defines the price -// policy for bn const operation(used after istanbul) -type parityChainSpecAltBnConstOperationPricing struct { - Price uint64 `json:"price"` -} - -// parityChainSepcAltBnPairingPricing defines the price policy -// for bn pairing. -type parityChainSepcAltBnPairingPricing struct { - Base uint64 `json:"base"` - Pair uint64 `json:"pair"` -} - -// parityChainSpecBlakePricing defines the price policy for blake2 f -// compression. -type parityChainSpecBlakePricing struct { - GasPerRound uint64 `json:"gas_per_round"` -} - -type parityChainSpecAlternativePrice struct { - AltBnConstOperationPrice *parityChainSpecAltBnConstOperationPricing `json:"alt_bn128_const_operations,omitempty"` - AltBnPairingPrice *parityChainSepcAltBnPairingPricing `json:"alt_bn128_pairing,omitempty"` -} - -// parityChainSpecVersionedPricing represents a single version price policy. -type parityChainSpecVersionedPricing struct { - Price *parityChainSpecAlternativePrice `json:"price,omitempty"` - Info string `json:"info,omitempty"` -} - -// newParityChainSpec converts a go-ethereum genesis block into a Parity specific -// chain specification format. -func newParityChainSpec(network string, genesis *core.Genesis, bootnodes []string) (*parityChainSpec, error) { - // Only ethash is currently supported between go-ethereum and Parity - if genesis.Config.Ethash == nil { - return nil, errors.New("unsupported consensus engine") - } - // Reconstruct the chain spec in Parity's format - spec := &parityChainSpec{ - Name: network, - Nodes: bootnodes, - Datadir: strings.ToLower(network), - } - spec.Engine.Ethash.Params.BlockReward = make(map[string]string) - spec.Engine.Ethash.Params.DifficultyBombDelays = make(map[string]string) - // Frontier - spec.Engine.Ethash.Params.MinimumDifficulty = (*hexutil.Big)(params.MinimumDifficulty) - spec.Engine.Ethash.Params.DifficultyBoundDivisor = (*hexutil.Big)(params.DifficultyBoundDivisor) - spec.Engine.Ethash.Params.DurationLimit = (*hexutil.Big)(params.DurationLimit) - spec.Engine.Ethash.Params.BlockReward["0x0"] = hexutil.EncodeBig(ethash.FrontierBlockReward) - - // Homestead - spec.Engine.Ethash.Params.HomesteadTransition = hexutil.Uint64(genesis.Config.HomesteadBlock.Uint64()) - - // Tangerine Whistle : 150 - // https://github.com/ethereum/EIPs/blob/master/EIPS/eip-608.md - spec.Params.EIP150Transition = hexutil.Uint64(genesis.Config.EIP150Block.Uint64()) - - // Spurious Dragon: 155, 160, 161, 170 - // https://github.com/ethereum/EIPs/blob/master/EIPS/eip-607.md - spec.Params.EIP155Transition = hexutil.Uint64(genesis.Config.EIP155Block.Uint64()) - spec.Params.EIP160Transition = hexutil.Uint64(genesis.Config.EIP155Block.Uint64()) - spec.Params.EIP161abcTransition = hexutil.Uint64(genesis.Config.EIP158Block.Uint64()) - spec.Params.EIP161dTransition = hexutil.Uint64(genesis.Config.EIP158Block.Uint64()) - - // Byzantium - if num := genesis.Config.ByzantiumBlock; num != nil { - spec.setByzantium(num) - } - // Constantinople - if num := genesis.Config.ConstantinopleBlock; num != nil { - spec.setConstantinople(num) - } - // ConstantinopleFix (remove eip-1283) - if num := genesis.Config.PetersburgBlock; num != nil { - spec.setConstantinopleFix(num) - } - // Istanbul - if num := genesis.Config.IstanbulBlock; num != nil { - spec.setIstanbul(num) - } - spec.Params.MaximumExtraDataSize = (hexutil.Uint64)(params.MaximumExtraDataSize) - spec.Params.MinGasLimit = (hexutil.Uint64)(params.MinGasLimit) - spec.Params.GasLimitBoundDivisor = (math2.HexOrDecimal64)(params.GasLimitBoundDivisor) - spec.Params.NetworkID = (hexutil.Uint64)(genesis.Config.ChainID.Uint64()) - spec.Params.ChainID = (hexutil.Uint64)(genesis.Config.ChainID.Uint64()) - spec.Params.MaxCodeSize = params.MaxCodeSize - // geth has it set from zero - spec.Params.MaxCodeSizeTransition = 0 - - // Disable this one - spec.Params.EIP98Transition = math.MaxInt64 - - spec.Genesis.Seal.Ethereum.Nonce = types.EncodeNonce(genesis.Nonce) - spec.Genesis.Seal.Ethereum.MixHash = genesis.Mixhash[:] - spec.Genesis.Difficulty = (*hexutil.Big)(genesis.Difficulty) - spec.Genesis.Author = genesis.Coinbase - spec.Genesis.Timestamp = (hexutil.Uint64)(genesis.Timestamp) - spec.Genesis.ParentHash = genesis.ParentHash - spec.Genesis.ExtraData = genesis.ExtraData - spec.Genesis.GasLimit = (hexutil.Uint64)(genesis.GasLimit) - - spec.Accounts = make(map[common.UnprefixedAddress]*parityChainSpecAccount) - for address, account := range genesis.Alloc { - bal := math2.HexOrDecimal256(*account.Balance) - - spec.Accounts[common.UnprefixedAddress(address)] = &parityChainSpecAccount{ - Balance: bal, - Nonce: math2.HexOrDecimal64(account.Nonce), - } - } - spec.setPrecompile(1, &parityChainSpecBuiltin{Name: "ecrecover", - Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 3000}}}) - - spec.setPrecompile(2, &parityChainSpecBuiltin{ - Name: "sha256", Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 60, Word: 12}}, - }) - spec.setPrecompile(3, &parityChainSpecBuiltin{ - Name: "ripemd160", Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 600, Word: 120}}, - }) - spec.setPrecompile(4, &parityChainSpecBuiltin{ - Name: "identity", Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 15, Word: 3}}, - }) - if genesis.Config.ByzantiumBlock != nil { - spec.setPrecompile(5, &parityChainSpecBuiltin{ - Name: "modexp", - ActivateAt: (*hexutil.Big)(genesis.Config.ByzantiumBlock), - Pricing: &parityChainSpecPricing{ - ModExp: &parityChainSpecModExpPricing{Divisor: 20}, - }, - }) - spec.setPrecompile(6, &parityChainSpecBuiltin{ - Name: "alt_bn128_add", - ActivateAt: (*hexutil.Big)(genesis.Config.ByzantiumBlock), - Pricing: &parityChainSpecPricing{ - Linear: &parityChainSpecLinearPricing{Base: 500, Word: 0}, - }, - }) - spec.setPrecompile(7, &parityChainSpecBuiltin{ - Name: "alt_bn128_mul", - ActivateAt: (*hexutil.Big)(genesis.Config.ByzantiumBlock), - Pricing: &parityChainSpecPricing{ - Linear: &parityChainSpecLinearPricing{Base: 40000, Word: 0}, - }, - }) - spec.setPrecompile(8, &parityChainSpecBuiltin{ - Name: "alt_bn128_pairing", - ActivateAt: (*hexutil.Big)(genesis.Config.ByzantiumBlock), - Pricing: &parityChainSpecPricing{ - AltBnPairing: &parityChainSepcAltBnPairingPricing{Base: 100000, Pair: 80000}, - }, - }) - } - if genesis.Config.IstanbulBlock != nil { - if genesis.Config.ByzantiumBlock == nil { - return nil, errors.New("invalid genesis, istanbul fork is enabled while byzantium is not") - } - spec.setPrecompile(6, &parityChainSpecBuiltin{ - Name: "alt_bn128_add", - ActivateAt: (*hexutil.Big)(genesis.Config.ByzantiumBlock), - Pricing: map[*hexutil.Big]*parityChainSpecVersionedPricing{ - (*hexutil.Big)(big.NewInt(0)): { - Price: &parityChainSpecAlternativePrice{ - AltBnConstOperationPrice: &parityChainSpecAltBnConstOperationPricing{Price: 500}, - }, - }, - (*hexutil.Big)(genesis.Config.IstanbulBlock): { - Price: &parityChainSpecAlternativePrice{ - AltBnConstOperationPrice: &parityChainSpecAltBnConstOperationPricing{Price: 150}, - }, - }, - }, - }) - spec.setPrecompile(7, &parityChainSpecBuiltin{ - Name: "alt_bn128_mul", - ActivateAt: (*hexutil.Big)(genesis.Config.ByzantiumBlock), - Pricing: map[*hexutil.Big]*parityChainSpecVersionedPricing{ - (*hexutil.Big)(big.NewInt(0)): { - Price: &parityChainSpecAlternativePrice{ - AltBnConstOperationPrice: &parityChainSpecAltBnConstOperationPricing{Price: 40000}, - }, - }, - (*hexutil.Big)(genesis.Config.IstanbulBlock): { - Price: &parityChainSpecAlternativePrice{ - AltBnConstOperationPrice: &parityChainSpecAltBnConstOperationPricing{Price: 6000}, - }, - }, - }, - }) - spec.setPrecompile(8, &parityChainSpecBuiltin{ - Name: "alt_bn128_pairing", - ActivateAt: (*hexutil.Big)(genesis.Config.ByzantiumBlock), - Pricing: map[*hexutil.Big]*parityChainSpecVersionedPricing{ - (*hexutil.Big)(big.NewInt(0)): { - Price: &parityChainSpecAlternativePrice{ - AltBnPairingPrice: &parityChainSepcAltBnPairingPricing{Base: 100000, Pair: 80000}, - }, - }, - (*hexutil.Big)(genesis.Config.IstanbulBlock): { - Price: &parityChainSpecAlternativePrice{ - AltBnPairingPrice: &parityChainSepcAltBnPairingPricing{Base: 45000, Pair: 34000}, - }, - }, - }, - }) - spec.setPrecompile(9, &parityChainSpecBuiltin{ - Name: "blake2_f", - ActivateAt: (*hexutil.Big)(genesis.Config.IstanbulBlock), - Pricing: &parityChainSpecPricing{ - Blake2F: &parityChainSpecBlakePricing{GasPerRound: 1}, - }, - }) - } - return spec, nil -} - -func (spec *parityChainSpec) setPrecompile(address byte, data *parityChainSpecBuiltin) { - if spec.Accounts == nil { - spec.Accounts = make(map[common.UnprefixedAddress]*parityChainSpecAccount) - } - a := common.UnprefixedAddress(common.BytesToAddress([]byte{address})) - if _, exist := spec.Accounts[a]; !exist { - spec.Accounts[a] = &parityChainSpecAccount{} - } - spec.Accounts[a].Builtin = data -} - -func (spec *parityChainSpec) setByzantium(num *big.Int) { - spec.Engine.Ethash.Params.BlockReward[hexutil.EncodeBig(num)] = hexutil.EncodeBig(ethash.ByzantiumBlockReward) - spec.Engine.Ethash.Params.DifficultyBombDelays[hexutil.EncodeBig(num)] = hexutil.EncodeUint64(3000000) - n := hexutil.Uint64(num.Uint64()) - spec.Engine.Ethash.Params.EIP100bTransition = n - spec.Params.EIP140Transition = n - spec.Params.EIP211Transition = n - spec.Params.EIP214Transition = n - spec.Params.EIP658Transition = n -} - -func (spec *parityChainSpec) setConstantinople(num *big.Int) { - spec.Engine.Ethash.Params.BlockReward[hexutil.EncodeBig(num)] = hexutil.EncodeBig(ethash.ConstantinopleBlockReward) - spec.Engine.Ethash.Params.DifficultyBombDelays[hexutil.EncodeBig(num)] = hexutil.EncodeUint64(2000000) - n := hexutil.Uint64(num.Uint64()) - spec.Params.EIP145Transition = n - spec.Params.EIP1014Transition = n - spec.Params.EIP1052Transition = n - spec.Params.EIP1283Transition = n -} - -func (spec *parityChainSpec) setConstantinopleFix(num *big.Int) { - spec.Params.EIP1283DisableTransition = hexutil.Uint64(num.Uint64()) -} - -func (spec *parityChainSpec) setIstanbul(num *big.Int) { - spec.Params.EIP1344Transition = hexutil.Uint64(num.Uint64()) - spec.Params.EIP1884Transition = hexutil.Uint64(num.Uint64()) - spec.Params.EIP2028Transition = hexutil.Uint64(num.Uint64()) - spec.Params.EIP1283ReenableTransition = hexutil.Uint64(num.Uint64()) -} - -// pyEthereumGenesisSpec represents the genesis specification format used by the -// Python Ethereum implementation. -type pyEthereumGenesisSpec struct { - Nonce types.BlockNonce `json:"nonce"` - Timestamp hexutil.Uint64 `json:"timestamp"` - ExtraData hexutil.Bytes `json:"extraData"` - GasLimit hexutil.Uint64 `json:"gasLimit"` - Difficulty *hexutil.Big `json:"difficulty"` - Mixhash common.Hash `json:"mixhash"` - Coinbase common.Address `json:"coinbase"` - Alloc core.GenesisAlloc `json:"alloc"` - ParentHash common.Hash `json:"parentHash"` -} - -// newPyEthereumGenesisSpec converts a go-ethereum genesis block into a Parity specific -// chain specification format. -func newPyEthereumGenesisSpec(network string, genesis *core.Genesis) (*pyEthereumGenesisSpec, error) { - // Only ethash is currently supported between go-ethereum and pyethereum - if genesis.Config.Ethash == nil { - return nil, errors.New("unsupported consensus engine") - } - spec := &pyEthereumGenesisSpec{ - Nonce: types.EncodeNonce(genesis.Nonce), - Timestamp: (hexutil.Uint64)(genesis.Timestamp), - ExtraData: genesis.ExtraData, - GasLimit: (hexutil.Uint64)(genesis.GasLimit), - Difficulty: (*hexutil.Big)(genesis.Difficulty), - Mixhash: genesis.Mixhash, - Coinbase: genesis.Coinbase, - Alloc: genesis.Alloc, - ParentHash: genesis.ParentHash, - } - return spec, nil -} diff --git a/cmd/puppeth/genesis_test.go b/cmd/puppeth/genesis_test.go deleted file mode 100644 index 605c1070a80cb..0000000000000 --- a/cmd/puppeth/genesis_test.go +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright 2018 The go-ethereum Authors -// This file is part of go-ethereum. -// -// go-ethereum is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// go-ethereum is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with go-ethereum. If not, see . - -package main - -import ( - "bytes" - "encoding/json" - "os" - "reflect" - "strings" - "testing" - - "github.com/davecgh/go-spew/spew" - "github.com/ethereum/go-ethereum/core" -) - -// Tests the go-ethereum to Aleth chainspec conversion for the Stureby testnet. -func TestAlethSturebyConverter(t *testing.T) { - blob, err := os.ReadFile("testdata/stureby_geth.json") - if err != nil { - t.Fatalf("could not read file: %v", err) - } - var genesis core.Genesis - if err := json.Unmarshal(blob, &genesis); err != nil { - t.Fatalf("failed parsing genesis: %v", err) - } - spec, err := newAlethGenesisSpec("stureby", &genesis) - if err != nil { - t.Fatalf("failed creating chainspec: %v", err) - } - - expBlob, err := os.ReadFile("testdata/stureby_aleth.json") - if err != nil { - t.Fatalf("could not read file: %v", err) - } - expspec := &alethGenesisSpec{} - if err := json.Unmarshal(expBlob, expspec); err != nil { - t.Fatalf("failed parsing genesis: %v", err) - } - if !reflect.DeepEqual(expspec, spec) { - t.Errorf("chainspec mismatch") - c := spew.ConfigState{ - DisablePointerAddresses: true, - SortKeys: true, - } - exp := strings.Split(c.Sdump(expspec), "\n") - got := strings.Split(c.Sdump(spec), "\n") - for i := 0; i < len(exp) && i < len(got); i++ { - if exp[i] != got[i] { - t.Logf("got: %v\nexp: %v\n", exp[i], got[i]) - } - } - } -} - -// Tests the go-ethereum to Parity chainspec conversion for the Stureby testnet. -func TestParitySturebyConverter(t *testing.T) { - blob, err := os.ReadFile("testdata/stureby_geth.json") - if err != nil { - t.Fatalf("could not read file: %v", err) - } - var genesis core.Genesis - if err := json.Unmarshal(blob, &genesis); err != nil { - t.Fatalf("failed parsing genesis: %v", err) - } - spec, err := newParityChainSpec("stureby", &genesis, []string{}) - if err != nil { - t.Fatalf("failed creating chainspec: %v", err) - } - enc, err := json.MarshalIndent(spec, "", " ") - if err != nil { - t.Fatalf("failed encoding chainspec: %v", err) - } - expBlob, err := os.ReadFile("testdata/stureby_parity.json") - if err != nil { - t.Fatalf("could not read file: %v", err) - } - if !bytes.Equal(expBlob, enc) { - t.Fatalf("chainspec mismatch") - } -} diff --git a/cmd/puppeth/module.go b/cmd/puppeth/module.go index b6a029a01a48a..771ae38058bc9 100644 --- a/cmd/puppeth/module.go +++ b/cmd/puppeth/module.go @@ -150,3 +150,12 @@ func checkPort(host string, port int) error { conn.Close() return nil } + +// getEthName gets the Ethereum Name from ethstats +func getEthName(s string) string { + n := strings.Index(s, ":") + if n >= 0 { + return s[:n] + } + return s +} diff --git a/cmd/puppeth/module_dashboard.go b/cmd/puppeth/module_dashboard.go index 35cfada66fd3f..fbbbb66501a7d 100644 --- a/cmd/puppeth/module_dashboard.go +++ b/cmd/puppeth/module_dashboard.go @@ -18,7 +18,6 @@ package main import ( "bytes" - "encoding/json" "fmt" "html/template" "math/rand" @@ -582,36 +581,6 @@ func deployDashboard(client *sshClient, network string, conf *config, config *da // Marshal the genesis spec files for go-ethereum and all the other clients genesis, _ := conf.Genesis.MarshalJSON() files[filepath.Join(workdir, network+".json")] = genesis - - if conf.Genesis.Config.Ethash != nil { - cppSpec, err := newAlethGenesisSpec(network, conf.Genesis) - if err != nil { - return nil, err - } - cppSpecJSON, _ := json.Marshal(cppSpec) - files[filepath.Join(workdir, network+"-cpp.json")] = cppSpecJSON - - harmonySpecJSON, _ := conf.Genesis.MarshalJSON() - files[filepath.Join(workdir, network+"-harmony.json")] = harmonySpecJSON - - paritySpec, err := newParityChainSpec(network, conf.Genesis, conf.bootnodes) - if err != nil { - return nil, err - } - paritySpecJSON, _ := json.Marshal(paritySpec) - files[filepath.Join(workdir, network+"-parity.json")] = paritySpecJSON - - pyethSpec, err := newPyEthereumGenesisSpec(network, conf.Genesis) - if err != nil { - return nil, err - } - pyethSpecJSON, _ := json.Marshal(pyethSpec) - files[filepath.Join(workdir, network+"-python.json")] = pyethSpecJSON - } else { - for _, client := range []string{"cpp", "harmony", "parity", "python"} { - files[filepath.Join(workdir, network+"-"+client+".json")] = []byte{} - } - } files[filepath.Join(workdir, "puppeth.png")] = dashboardMascot // Upload the deployment files to the remote server (and clean up afterwards) diff --git a/cmd/puppeth/module_explorer.go b/cmd/puppeth/module_explorer.go index 1165f70fcf516..3812f9fdb9637 100644 --- a/cmd/puppeth/module_explorer.go +++ b/cmd/puppeth/module_explorer.go @@ -104,7 +104,7 @@ func deployExplorer(client *sshClient, network string, bootnodes []string, confi "Datadir": config.node.datadir, "DBDir": config.dbdir, "EthPort": config.node.port, - "EthName": config.node.ethstats[:strings.Index(config.node.ethstats, ":")], + "EthName": getEthName(config.node.ethstats), "WebPort": config.port, "Transformer": transformer, }) diff --git a/cmd/puppeth/module_faucet.go b/cmd/puppeth/module_faucet.go index 88cb80ae4c42c..a4f6e65694df4 100644 --- a/cmd/puppeth/module_faucet.go +++ b/cmd/puppeth/module_faucet.go @@ -116,7 +116,7 @@ func deployFaucet(client *sshClient, network string, bootnodes []string, config "VHost": config.host, "ApiPort": config.port, "EthPort": config.node.port, - "EthName": config.node.ethstats[:strings.Index(config.node.ethstats, ":")], + "EthName": getEthName(config.node.ethstats), "CaptchaToken": config.captchaToken, "CaptchaSecret": config.captchaSecret, "FaucetAmount": config.amount, diff --git a/cmd/puppeth/module_node.go b/cmd/puppeth/module_node.go index 3ea96870d4f5d..734dd04051284 100644 --- a/cmd/puppeth/module_node.go +++ b/cmd/puppeth/module_node.go @@ -42,7 +42,7 @@ ADD genesis.json /genesis.json RUN \ echo 'geth --cache 512 init /genesis.json' > geth.sh && \{{if .Unlock}} echo 'mkdir -p /root/.ethereum/keystore/ && cp /signer.json /root/.ethereum/keystore/' >> geth.sh && \{{end}} - echo $'exec geth --networkid {{.NetworkID}} --cache 512 --port {{.Port}} --nat extip:{{.IP}} --maxpeers {{.Peers}} {{.LightFlag}} --ethstats \'{{.Ethstats}}\' {{if .Bootnodes}}--bootnodes {{.Bootnodes}}{{end}} {{if .Etherbase}}--miner.etherbase {{.Etherbase}} --mine --miner.threads 1{{end}} {{if .Unlock}}--unlock 0 --password /signer.pass --mine{{end}} --miner.gastarget {{.GasTarget}} --miner.gaslimit {{.GasLimit}} --miner.gasprice {{.GasPrice}}' >> geth.sh + echo $'exec geth --networkid {{.NetworkID}} --cache 512 --port {{.Port}} --nat extip:{{.IP}} --maxpeers {{.Peers}} {{.LightFlag}} --ethstats \'{{.Ethstats}}\' {{if .Bootnodes}}--bootnodes {{.Bootnodes}}{{end}} {{if .Etherbase}}--miner.etherbase {{.Etherbase}} --mine --miner.threads 1{{end}} {{if .Unlock}}--unlock 0 --password /signer.pass --mine{{end}} --miner.gaslimit {{.GasLimit}} --miner.gasprice {{.GasPrice}}' >> geth.sh ENTRYPOINT ["/bin/sh", "geth.sh"] ` @@ -68,7 +68,6 @@ services: - LIGHT_PEERS={{.LightPeers}} - STATS_NAME={{.Ethstats}} - MINER_NAME={{.Etherbase}} - - GAS_TARGET={{.GasTarget}} - GAS_LIMIT={{.GasLimit}} - GAS_PRICE={{.GasPrice}} logging: @@ -106,7 +105,6 @@ func deployNode(client *sshClient, network string, bootnodes []string, config *n "Bootnodes": strings.Join(bootnodes, ","), "Ethstats": config.ethstats, "Etherbase": config.etherbase, - "GasTarget": uint64(1000000 * config.gasTarget), "GasLimit": uint64(1000000 * config.gasLimit), "GasPrice": uint64(1000000000 * config.gasPrice), "Unlock": config.keyJSON != "", @@ -123,9 +121,8 @@ func deployNode(client *sshClient, network string, bootnodes []string, config *n "TotalPeers": config.peersTotal, "Light": config.peersLight > 0, "LightPeers": config.peersLight, - "Ethstats": config.ethstats[:strings.Index(config.ethstats, ":")], + "Ethstats": getEthName(config.ethstats), "Etherbase": config.etherbase, - "GasTarget": config.gasTarget, "GasLimit": config.gasLimit, "GasPrice": config.gasPrice, }) @@ -164,7 +161,6 @@ type nodeInfos struct { etherbase string keyJSON string keyPass string - gasTarget float64 gasLimit float64 gasPrice float64 } @@ -179,10 +175,9 @@ func (info *nodeInfos) Report() map[string]string { "Peer count (light nodes)": strconv.Itoa(info.peersLight), "Ethstats username": info.ethstats, } - if info.gasTarget > 0 { + if info.gasLimit > 0 { // Miner or signer node report["Gas price (minimum accepted)"] = fmt.Sprintf("%0.3f GWei", info.gasPrice) - report["Gas floor (baseline target)"] = fmt.Sprintf("%0.3f MGas", info.gasTarget) report["Gas ceil (target maximum)"] = fmt.Sprintf("%0.3f MGas", info.gasLimit) if info.etherbase != "" { @@ -223,7 +218,6 @@ func checkNode(client *sshClient, network string, boot bool) (*nodeInfos, error) // Resolve a few types from the environmental variables totalPeers, _ := strconv.Atoi(infos.envvars["TOTAL_PEERS"]) lightPeers, _ := strconv.Atoi(infos.envvars["LIGHT_PEERS"]) - gasTarget, _ := strconv.ParseFloat(infos.envvars["GAS_TARGET"], 64) gasLimit, _ := strconv.ParseFloat(infos.envvars["GAS_LIMIT"], 64) gasPrice, _ := strconv.ParseFloat(infos.envvars["GAS_PRICE"], 64) @@ -263,7 +257,6 @@ func checkNode(client *sshClient, network string, boot bool) (*nodeInfos, error) etherbase: infos.envvars["MINER_NAME"], keyJSON: keyJSON, keyPass: keyPass, - gasTarget: gasTarget, gasLimit: gasLimit, gasPrice: gasPrice, } diff --git a/cmd/puppeth/puppeth.go b/cmd/puppeth/puppeth.go index c3de5f9360242..415542b60cc9f 100644 --- a/cmd/puppeth/puppeth.go +++ b/cmd/puppeth/puppeth.go @@ -24,7 +24,7 @@ import ( "time" "github.com/ethereum/go-ethereum/log" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) // main is just a boring entry point to set up the CLI app. @@ -33,11 +33,11 @@ func main() { app.Name = "puppeth" app.Usage = "assemble and maintain private Ethereum networks" app.Flags = []cli.Flag{ - cli.StringFlag{ + &cli.StringFlag{ Name: "network", Usage: "name of the network to administer (no spaces or hyphens, please)", }, - cli.IntFlag{ + &cli.IntFlag{ Name: "loglevel", Value: 3, Usage: "log level to emit to the screen", diff --git a/cmd/puppeth/ssh.go b/cmd/puppeth/ssh.go index 95a36f327236b..a20b3bfda2095 100644 --- a/cmd/puppeth/ssh.go +++ b/cmd/puppeth/ssh.go @@ -30,7 +30,7 @@ import ( "github.com/ethereum/go-ethereum/log" "golang.org/x/crypto/ssh" "golang.org/x/crypto/ssh/agent" - "golang.org/x/crypto/ssh/terminal" + "golang.org/x/term" ) // sshClient is a small wrapper around Go's SSH client with a few utility methods @@ -101,7 +101,7 @@ func dial(server string, pubkey []byte) (*sshClient, error) { key, err := ssh.ParsePrivateKey(buf) if err != nil { fmt.Printf("What's the decryption password for %s? (won't be echoed)\n>", path) - blob, err := terminal.ReadPassword(int(os.Stdin.Fd())) + blob, err := term.ReadPassword(int(os.Stdin.Fd())) fmt.Println() if err != nil { log.Warn("Couldn't read password", "err", err) @@ -118,7 +118,7 @@ func dial(server string, pubkey []byte) (*sshClient, error) { } auths = append(auths, ssh.PasswordCallback(func() (string, error) { fmt.Printf("What's the login password for %s at %s? (won't be echoed)\n> ", username, server) - blob, err := terminal.ReadPassword(int(os.Stdin.Fd())) + blob, err := term.ReadPassword(int(os.Stdin.Fd())) fmt.Println() return string(blob), err @@ -163,7 +163,7 @@ func dial(server string, pubkey []byte) (*sshClient, error) { return nil } // We have a mismatch, forbid connecting - return errors.New("ssh key mismatch, readd the machine to update") + return errors.New("ssh key mismatch, re-add the machine to update") } client, err := ssh.Dial("tcp", hostport, &ssh.ClientConfig{User: username, Auth: auths, HostKeyCallback: keycheck}) if err != nil { diff --git a/cmd/puppeth/wizard.go b/cmd/puppeth/wizard.go index f7aafd4dd90a9..6e5ca41d68fab 100644 --- a/cmd/puppeth/wizard.go +++ b/cmd/puppeth/wizard.go @@ -34,7 +34,7 @@ import ( "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/log" "github.com/peterh/liner" - "golang.org/x/crypto/ssh/terminal" + "golang.org/x/term" ) // config contains all the configurations needed by puppeth that should be saved @@ -228,7 +228,7 @@ func (w *wizard) readDefaultFloat(def float64) float64 { // line and returns it. The input will not be echoed. func (w *wizard) readPassword() string { fmt.Printf("> ") - text, err := terminal.ReadPassword(int(os.Stdin.Fd())) + text, err := term.ReadPassword(int(os.Stdin.Fd())) if err != nil { log.Crit("Failed to read password", "err", err) } diff --git a/cmd/puppeth/wizard_genesis.go b/cmd/puppeth/wizard_genesis.go index cb056ab13356a..ac17bc7b271cc 100644 --- a/cmd/puppeth/wizard_genesis.go +++ b/cmd/puppeth/wizard_genesis.go @@ -250,8 +250,8 @@ func (w *wizard) manageGenesis() { case "2": // Save whatever genesis configuration we currently have fmt.Println() - fmt.Printf("Which folder to save the genesis specs into? (default = current)\n") - fmt.Printf(" Will create %s.json, %s-aleth.json, %s-harmony.json, %s-parity.json\n", w.network, w.network, w.network, w.network) + fmt.Printf("Which folder to save the genesis spec into? (default = current)\n") + fmt.Printf(" Will create %s.json\n", w.network) folder := w.readDefaultString(".") if err := os.MkdirAll(folder, 0755); err != nil { @@ -268,21 +268,6 @@ func (w *wizard) manageGenesis() { } log.Info("Saved native genesis chain spec", "path", gethJson) - // Export the genesis spec used by Aleth (formerly C++ Ethereum) - if spec, err := newAlethGenesisSpec(w.network, w.conf.Genesis); err != nil { - log.Error("Failed to create Aleth chain spec", "err", err) - } else { - saveGenesis(folder, w.network, "aleth", spec) - } - // Export the genesis spec used by Parity - if spec, err := newParityChainSpec(w.network, w.conf.Genesis, []string{}); err != nil { - log.Error("Failed to create Parity chain spec", "err", err) - } else { - saveGenesis(folder, w.network, "parity", spec) - } - // Export the genesis spec used by Harmony (formerly EthereumJ) - saveGenesis(folder, w.network, "harmony", w.conf.Genesis) - case "3": // Make sure we don't have any services running if len(w.conf.servers()) > 0 { @@ -298,15 +283,3 @@ func (w *wizard) manageGenesis() { return } } - -// saveGenesis JSON encodes an arbitrary genesis spec into a pre-defined file. -func saveGenesis(folder, network, client string, spec interface{}) { - path := filepath.Join(folder, fmt.Sprintf("%s-%s.json", network, client)) - - out, _ := json.MarshalIndent(spec, "", " ") - if err := os.WriteFile(path, out, 0644); err != nil { - log.Error("Failed to save genesis file", "client", client, "err", err) - return - } - log.Info("Saved genesis chain spec", "client", client, "path", path) -} diff --git a/cmd/puppeth/wizard_node.go b/cmd/puppeth/wizard_node.go index 2bae332142835..c38750875aad9 100644 --- a/cmd/puppeth/wizard_node.go +++ b/cmd/puppeth/wizard_node.go @@ -50,7 +50,7 @@ func (w *wizard) deployNode(boot bool) { if boot { infos = &nodeInfos{port: 30303, peersTotal: 512, peersLight: 256} } else { - infos = &nodeInfos{port: 30303, peersTotal: 50, peersLight: 0, gasTarget: 7.5, gasLimit: 10, gasPrice: 1} + infos = &nodeInfos{port: 30303, peersTotal: 50, peersLight: 0, gasLimit: 10, gasPrice: 1} } } existed := err == nil @@ -148,10 +148,6 @@ func (w *wizard) deployNode(boot bool) { } } // Establish the gas dynamics to be enforced by the signer - fmt.Println() - fmt.Printf("What gas limit should empty blocks target (MGas)? (default = %0.3f)\n", infos.gasTarget) - infos.gasTarget = w.readDefaultFloat(infos.gasTarget) - fmt.Println() fmt.Printf("What gas limit should full blocks target (MGas)? (default = %0.3f)\n", infos.gasLimit) infos.gasLimit = w.readDefaultFloat(infos.gasLimit) diff --git a/cmd/rlpdump/main.go b/cmd/rlpdump/main.go index 9c0af012480fe..70337749aea30 100644 --- a/cmd/rlpdump/main.go +++ b/cmd/rlpdump/main.go @@ -83,7 +83,7 @@ func main() { if err != nil { die(err) } - fmt.Printf("0x%x\n", data) + fmt.Printf("%#x\n", data) return } else { err := rlpToText(r, out) diff --git a/cmd/rlpdump/rlpdump_test.go b/cmd/rlpdump/rlpdump_test.go index 899beef32f4a6..a9ab57fdb880e 100644 --- a/cmd/rlpdump/rlpdump_test.go +++ b/cmd/rlpdump/rlpdump_test.go @@ -43,7 +43,7 @@ func TestRoundtrip(t *testing.T) { t.Errorf("test %d: error %v", i, err) continue } - have := fmt.Sprintf("0x%x", rlpBytes) + have := fmt.Sprintf("%#x", rlpBytes) if have != want { t.Errorf("test %d: have\n%v\nwant:\n%v\n", i, have, want) } diff --git a/cmd/utils/cmd.go b/cmd/utils/cmd.go index 47ad3b22c8dd1..90f0090414773 100644 --- a/cmd/utils/cmd.go +++ b/cmd/utils/cmd.go @@ -41,7 +41,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/rlp" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) const ( @@ -78,10 +78,10 @@ func StartNode(ctx *cli.Context, stack *node.Node, isConsole bool) { defer signal.Stop(sigc) minFreeDiskSpace := 2 * ethconfig.Defaults.TrieDirtyCache // Default 2 * 256Mb - if ctx.GlobalIsSet(MinFreeDiskSpaceFlag.Name) { - minFreeDiskSpace = ctx.GlobalInt(MinFreeDiskSpaceFlag.Name) - } else if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheGCFlag.Name) { - minFreeDiskSpace = 2 * ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100 + if ctx.IsSet(MinFreeDiskSpaceFlag.Name) { + minFreeDiskSpace = ctx.Int(MinFreeDiskSpaceFlag.Name) + } else if ctx.IsSet(CacheFlag.Name) || ctx.IsSet(CacheGCFlag.Name) { + minFreeDiskSpace = 2 * ctx.Int(CacheFlag.Name) * ctx.Int(CacheGCFlag.Name) / 100 } if minFreeDiskSpace > 0 { go monitorFreeDiskSpace(sigc, stack.InstanceDir(), uint64(minFreeDiskSpace)*1024*1024) diff --git a/cmd/utils/customflags.go b/cmd/utils/customflags.go deleted file mode 100644 index e5be085a5db7e..0000000000000 --- a/cmd/utils/customflags.go +++ /dev/null @@ -1,211 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of go-ethereum. -// -// go-ethereum is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// go-ethereum is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with go-ethereum. If not, see . - -package utils - -import ( - "encoding" - "errors" - "flag" - "math/big" - "os" - "os/user" - "path" - "strings" - - "github.com/ethereum/go-ethereum/common/math" - "gopkg.in/urfave/cli.v1" -) - -// Custom type which is registered in the flags library which cli uses for -// argument parsing. This allows us to expand Value to an absolute path when -// the argument is parsed -type DirectoryString string - -func (s *DirectoryString) String() string { - return string(*s) -} - -func (s *DirectoryString) Set(value string) error { - *s = DirectoryString(expandPath(value)) - return nil -} - -// Custom cli.Flag type which expand the received string to an absolute path. -// e.g. ~/.ethereum -> /home/username/.ethereum -type DirectoryFlag struct { - Name string - Value DirectoryString - Usage string - EnvVar string -} - -func (f DirectoryFlag) String() string { - return cli.FlagStringer(f) -} - -// called by cli library, grabs variable from environment (if in env) -// and adds variable to flag set for parsing. -func (f DirectoryFlag) Apply(set *flag.FlagSet) { - eachName(f.Name, func(name string) { - set.Var(&f.Value, f.Name, f.Usage) - }) -} - -func (f DirectoryFlag) GetName() string { - return f.Name -} - -func (f *DirectoryFlag) Set(value string) { - f.Value.Set(value) -} - -func eachName(longName string, fn func(string)) { - parts := strings.Split(longName, ",") - for _, name := range parts { - name = strings.Trim(name, " ") - fn(name) - } -} - -type TextMarshaler interface { - encoding.TextMarshaler - encoding.TextUnmarshaler -} - -// textMarshalerVal turns a TextMarshaler into a flag.Value -type textMarshalerVal struct { - v TextMarshaler -} - -func (v textMarshalerVal) String() string { - if v.v == nil { - return "" - } - text, _ := v.v.MarshalText() - return string(text) -} - -func (v textMarshalerVal) Set(s string) error { - return v.v.UnmarshalText([]byte(s)) -} - -// TextMarshalerFlag wraps a TextMarshaler value. -type TextMarshalerFlag struct { - Name string - Value TextMarshaler - Usage string - EnvVar string -} - -func (f TextMarshalerFlag) GetName() string { - return f.Name -} - -func (f TextMarshalerFlag) String() string { - return cli.FlagStringer(f) -} - -func (f TextMarshalerFlag) Apply(set *flag.FlagSet) { - eachName(f.Name, func(name string) { - set.Var(textMarshalerVal{f.Value}, f.Name, f.Usage) - }) -} - -// GlobalTextMarshaler returns the value of a TextMarshalerFlag from the global flag set. -func GlobalTextMarshaler(ctx *cli.Context, name string) TextMarshaler { - val := ctx.GlobalGeneric(name) - if val == nil { - return nil - } - return val.(textMarshalerVal).v -} - -// BigFlag is a command line flag that accepts 256 bit big integers in decimal or -// hexadecimal syntax. -type BigFlag struct { - Name string - Value *big.Int - Usage string - EnvVar string -} - -// bigValue turns *big.Int into a flag.Value -type bigValue big.Int - -func (b *bigValue) String() string { - if b == nil { - return "" - } - return (*big.Int)(b).String() -} - -func (b *bigValue) Set(s string) error { - intVal, ok := math.ParseBig256(s) - if !ok { - return errors.New("invalid integer syntax") - } - *b = (bigValue)(*intVal) - return nil -} - -func (f BigFlag) GetName() string { - return f.Name -} - -func (f BigFlag) String() string { - return cli.FlagStringer(f) -} - -func (f BigFlag) Apply(set *flag.FlagSet) { - eachName(f.Name, func(name string) { - f.Value = new(big.Int) - set.Var((*bigValue)(f.Value), f.Name, f.Usage) - }) -} - -// GlobalBig returns the value of a BigFlag from the global flag set. -func GlobalBig(ctx *cli.Context, name string) *big.Int { - val := ctx.GlobalGeneric(name) - if val == nil { - return nil - } - return (*big.Int)(val.(*bigValue)) -} - -// Expands a file path -// 1. replace tilde with users home dir -// 2. expands embedded environment variables -// 3. cleans the path, e.g. /a/b/../c -> /a/c -// Note, it has limitations, e.g. ~someuser/tmp will not be expanded -func expandPath(p string) string { - if strings.HasPrefix(p, "~/") || strings.HasPrefix(p, "~\\") { - if home := HomeDir(); home != "" { - p = home + p[1:] - } - } - return path.Clean(os.ExpandEnv(p)) -} - -func HomeDir() string { - if home := os.Getenv("HOME"); home != "" { - return home - } - if usr, err := user.Current(); err == nil { - return usr.HomeDir - } - return "" -} diff --git a/cmd/utils/diskusage.go b/cmd/utils/diskusage.go index 14cd5cd0bef86..0e88f91944302 100644 --- a/cmd/utils/diskusage.go +++ b/cmd/utils/diskusage.go @@ -33,6 +33,7 @@ func getFreeDiskSpace(path string) (uint64, error) { // Available blocks * size per block = available space in bytes var bavail = stat.Bavail + // nolint:staticcheck if stat.Bavail < 0 { // FreeBSD can have a negative number of blocks available // because of the grace limit. diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 0b28cd09f1418..ca6ded4756683 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -18,25 +18,24 @@ package utils import ( + "context" "crypto/ecdsa" + "errors" "fmt" - "io" "math" "math/big" + "net/http" "os" "path/filepath" godebug "runtime/debug" "strconv" "strings" - "text/tabwriter" - "text/template" "time" "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/accounts/keystore" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/fdlimit" - "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" @@ -46,6 +45,7 @@ import ( ethcatalyst "github.com/ethereum/go-ethereum/eth/catalyst" "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/ethconfig" + "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/eth/gasprice" "github.com/ethereum/go-ethereum/eth/tracers" "github.com/ethereum/go-ethereum/ethdb" @@ -67,39 +67,12 @@ import ( "github.com/ethereum/go-ethereum/p2p/nat" "github.com/ethereum/go-ethereum/p2p/netutil" "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/rpc" pcsclite "github.com/gballet/go-libpcsclite" gopsutil "github.com/shirou/gopsutil/mem" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) -func init() { - cli.AppHelpTemplate = `{{.Name}} {{if .Flags}}[global options] {{end}}command{{if .Flags}} [command options]{{end}} [arguments...] - -VERSION: - {{.Version}} - -COMMANDS: - {{range .Commands}}{{.Name}}{{with .ShortName}}, {{.}}{{end}}{{ "\t" }}{{.Usage}} - {{end}}{{if .Flags}} -GLOBAL OPTIONS: - {{range .Flags}}{{.}} - {{end}}{{end}} -` - cli.CommandHelpTemplate = flags.CommandHelpTemplate - cli.HelpPrinter = printHelp -} - -func printHelp(out io.Writer, templ string, data interface{}) { - funcMap := template.FuncMap{"join": strings.Join} - t := template.Must(template.New("help").Funcs(funcMap).Parse(templ)) - w := tabwriter.NewWriter(out, 38, 8, 2, ' ', 0) - err := t.Execute(w, data) - if err != nil { - panic(err) - } - w.Flush() -} - // These are all the command line flags we support. // If you add to this list, please remember to include the // flag in the appropriate command definition. @@ -109,725 +82,909 @@ func printHelp(out io.Writer, templ string, data interface{}) { var ( // General settings - DataDirFlag = DirectoryFlag{ - Name: "datadir", - Usage: "Data directory for the databases and keystore", - Value: DirectoryString(node.DefaultDataDir()), - } - RemoteDBFlag = cli.StringFlag{ - Name: "remotedb", - Usage: "URL for remote database", - } - AncientFlag = DirectoryFlag{ - Name: "datadir.ancient", - Usage: "Data directory for ancient chain segments (default = inside chaindata)", - } - MinFreeDiskSpaceFlag = DirectoryFlag{ - Name: "datadir.minfreedisk", - Usage: "Minimum free disk space in MB, once reached triggers auto shut down (default = --cache.gc converted to MB, 0 = disabled)", - } - KeyStoreDirFlag = DirectoryFlag{ - Name: "keystore", - Usage: "Directory for the keystore (default = inside the datadir)", - } - USBFlag = cli.BoolFlag{ - Name: "usb", - Usage: "Enable monitoring and management of USB hardware wallets", - } - SmartCardDaemonPathFlag = cli.StringFlag{ - Name: "pcscdpath", - Usage: "Path to the smartcard daemon (pcscd) socket file", - Value: pcsclite.PCSCDSockName, - } - NetworkIdFlag = cli.Uint64Flag{ - Name: "networkid", - Usage: "Explicitly set network id (integer)(For testnets: use --ropsten, --rinkeby, --goerli instead)", - Value: ethconfig.Defaults.NetworkId, - } - MainnetFlag = cli.BoolFlag{ - Name: "mainnet", - Usage: "Ethereum mainnet", - } - RopstenFlag = cli.BoolFlag{ - Name: "ropsten", - Usage: "Ropsten network: pre-configured proof-of-work test network", - } - RinkebyFlag = cli.BoolFlag{ - Name: "rinkeby", - Usage: "Rinkeby network: pre-configured proof-of-authority test network", - } - GoerliFlag = cli.BoolFlag{ - Name: "goerli", - Usage: "Görli network: pre-configured proof-of-authority test network", - } - SepoliaFlag = cli.BoolFlag{ - Name: "sepolia", - Usage: "Sepolia network: pre-configured proof-of-work test network", - } - KilnFlag = cli.BoolFlag{ - Name: "kiln", - Usage: "Kiln network: pre-configured proof-of-work to proof-of-stake test network", - } - DeveloperFlag = cli.BoolFlag{ - Name: "dev", - Usage: "Ephemeral proof-of-authority network with a pre-funded developer account, mining enabled", - } - DeveloperPeriodFlag = cli.IntFlag{ - Name: "dev.period", - Usage: "Block period to use in developer mode (0 = mine only if transaction pending)", - } - DeveloperGasLimitFlag = cli.Uint64Flag{ - Name: "dev.gaslimit", - Usage: "Initial block gas limit", - Value: 11500000, - } - IdentityFlag = cli.StringFlag{ - Name: "identity", - Usage: "Custom node name", - } - DocRootFlag = DirectoryFlag{ - Name: "docroot", - Usage: "Document Root for HTTPClient file scheme", - Value: DirectoryString(HomeDir()), - } - ExitWhenSyncedFlag = cli.BoolFlag{ - Name: "exitwhensynced", - Usage: "Exits after block synchronisation completes", - } - IterativeOutputFlag = cli.BoolTFlag{ + DataDirFlag = &flags.DirectoryFlag{ + Name: "datadir", + Usage: "Data directory for the databases and keystore", + Value: flags.DirectoryString(node.DefaultDataDir()), + Category: flags.EthCategory, + } + RemoteDBFlag = &cli.StringFlag{ + Name: "remotedb", + Usage: "URL for remote database", + Category: flags.LoggingCategory, + } + AncientFlag = &flags.DirectoryFlag{ + Name: "datadir.ancient", + Usage: "Root directory for ancient data (default = inside chaindata)", + Category: flags.EthCategory, + } + MinFreeDiskSpaceFlag = &flags.DirectoryFlag{ + Name: "datadir.minfreedisk", + Usage: "Minimum free disk space in MB, once reached triggers auto shut down (default = --cache.gc converted to MB, 0 = disabled)", + Category: flags.EthCategory, + } + KeyStoreDirFlag = &flags.DirectoryFlag{ + Name: "keystore", + Usage: "Directory for the keystore (default = inside the datadir)", + Category: flags.AccountCategory, + } + USBFlag = &cli.BoolFlag{ + Name: "usb", + Usage: "Enable monitoring and management of USB hardware wallets", + Category: flags.AccountCategory, + } + SmartCardDaemonPathFlag = &cli.StringFlag{ + Name: "pcscdpath", + Usage: "Path to the smartcard daemon (pcscd) socket file", + Value: pcsclite.PCSCDSockName, + Category: flags.AccountCategory, + } + NetworkIdFlag = &cli.Uint64Flag{ + Name: "networkid", + Usage: "Explicitly set network id (integer)(For testnets: use --ropsten, --rinkeby, --goerli instead)", + Value: ethconfig.Defaults.NetworkId, + Category: flags.EthCategory, + } + MainnetFlag = &cli.BoolFlag{ + Name: "mainnet", + Usage: "Ethereum mainnet", + Category: flags.EthCategory, + } + RopstenFlag = &cli.BoolFlag{ + Name: "ropsten", + Usage: "Ropsten network: pre-configured proof-of-stake test network", + Category: flags.EthCategory, + } + RinkebyFlag = &cli.BoolFlag{ + Name: "rinkeby", + Usage: "Rinkeby network: pre-configured proof-of-authority test network", + Category: flags.EthCategory, + } + GoerliFlag = &cli.BoolFlag{ + Name: "goerli", + Usage: "Görli network: pre-configured proof-of-authority test network", + Category: flags.EthCategory, + } + SepoliaFlag = &cli.BoolFlag{ + Name: "sepolia", + Usage: "Sepolia network: pre-configured proof-of-work test network", + Category: flags.EthCategory, + } + KilnFlag = &cli.BoolFlag{ + Name: "kiln", + Usage: "Kiln network: pre-configured proof-of-work to proof-of-stake test network", + Category: flags.EthCategory, + } + + // Dev mode + DeveloperFlag = &cli.BoolFlag{ + Name: "dev", + Usage: "Ephemeral proof-of-authority network with a pre-funded developer account, mining enabled", + Category: flags.DevCategory, + } + DeveloperPeriodFlag = &cli.IntFlag{ + Name: "dev.period", + Usage: "Block period to use in developer mode (0 = mine only if transaction pending)", + Category: flags.DevCategory, + } + DeveloperGasLimitFlag = &cli.Uint64Flag{ + Name: "dev.gaslimit", + Usage: "Initial block gas limit", + Value: 11500000, + Category: flags.DevCategory, + } + + IdentityFlag = &cli.StringFlag{ + Name: "identity", + Usage: "Custom node name", + Category: flags.NetworkingCategory, + } + DocRootFlag = &flags.DirectoryFlag{ + Name: "docroot", + Usage: "Document Root for HTTPClient file scheme", + Value: flags.DirectoryString(flags.HomeDir()), + Category: flags.APICategory, + } + ExitWhenSyncedFlag = &cli.BoolFlag{ + Name: "exitwhensynced", + Usage: "Exits after block synchronisation completes", + Category: flags.EthCategory, + } + + // Dump command options. + IterativeOutputFlag = &cli.BoolFlag{ Name: "iterative", Usage: "Print streaming JSON iteratively, delimited by newlines", + Value: true, } - ExcludeStorageFlag = cli.BoolFlag{ + ExcludeStorageFlag = &cli.BoolFlag{ Name: "nostorage", Usage: "Exclude storage entries (save db lookups)", } - IncludeIncompletesFlag = cli.BoolFlag{ + IncludeIncompletesFlag = &cli.BoolFlag{ Name: "incompletes", Usage: "Include accounts for which we don't have the address (missing preimage)", } - ExcludeCodeFlag = cli.BoolFlag{ + ExcludeCodeFlag = &cli.BoolFlag{ Name: "nocode", Usage: "Exclude contract code (save db lookups)", } - StartKeyFlag = cli.StringFlag{ + StartKeyFlag = &cli.StringFlag{ Name: "start", Usage: "Start position. Either a hash or address", Value: "0x0000000000000000000000000000000000000000000000000000000000000000", } - DumpLimitFlag = cli.Uint64Flag{ + DumpLimitFlag = &cli.Uint64Flag{ Name: "limit", Usage: "Max number of elements (0 = no limit)", Value: 0, } + defaultSyncMode = ethconfig.Defaults.SyncMode - SyncModeFlag = TextMarshalerFlag{ - Name: "syncmode", - Usage: `Blockchain sync mode ("snap", "full" or "light")`, - Value: &defaultSyncMode, - } - GCModeFlag = cli.StringFlag{ - Name: "gcmode", - Usage: `Blockchain garbage collection mode ("full", "archive")`, - Value: "full", - } - SnapshotFlag = cli.BoolTFlag{ - Name: "snapshot", - Usage: `Enables snapshot-database mode (default = enable)`, - } - TxLookupLimitFlag = cli.Uint64Flag{ - Name: "txlookuplimit", - Usage: "Number of recent blocks to maintain transactions index for (default = about one year, 0 = entire chain)", - Value: ethconfig.Defaults.TxLookupLimit, - } - LightKDFFlag = cli.BoolFlag{ - Name: "lightkdf", - Usage: "Reduce key-derivation RAM & CPU usage at some expense of KDF strength", - } - EthRequiredBlocksFlag = cli.StringFlag{ - Name: "eth.requiredblocks", - Usage: "Comma separated block number-to-hash mappings to require for peering (=)", - } - LegacyWhitelistFlag = cli.StringFlag{ - Name: "whitelist", - Usage: "Comma separated block number-to-hash mappings to enforce (=) (deprecated in favor of --eth.requiredblocks)", - } - BloomFilterSizeFlag = cli.Uint64Flag{ - Name: "bloomfilter.size", - Usage: "Megabytes of memory allocated to bloom-filter for pruning", - Value: 2048, - } - OverrideArrowGlacierFlag = cli.Uint64Flag{ - Name: "override.arrowglacier", - Usage: "Manually specify Arrow Glacier fork-block, overriding the bundled setting", - } - OverrideTerminalTotalDifficulty = BigFlag{ - Name: "override.terminaltotaldifficulty", - Usage: "Manually specify TerminalTotalDifficulty, overriding the bundled setting", + SyncModeFlag = &flags.TextMarshalerFlag{ + Name: "syncmode", + Usage: `Blockchain sync mode ("snap", "full" or "light")`, + Value: &defaultSyncMode, + Category: flags.EthCategory, + } + GCModeFlag = &cli.StringFlag{ + Name: "gcmode", + Usage: `Blockchain garbage collection mode ("full", "archive")`, + Value: "full", + Category: flags.EthCategory, + } + SnapshotFlag = &cli.BoolFlag{ + Name: "snapshot", + Usage: `Enables snapshot-database mode (default = enable)`, + Value: true, + Category: flags.EthCategory, + } + TxLookupLimitFlag = &cli.Uint64Flag{ + Name: "txlookuplimit", + Usage: "Number of recent blocks to maintain transactions index for (default = about one year, 0 = entire chain)", + Value: ethconfig.Defaults.TxLookupLimit, + Category: flags.EthCategory, + } + LightKDFFlag = &cli.BoolFlag{ + Name: "lightkdf", + Usage: "Reduce key-derivation RAM & CPU usage at some expense of KDF strength", + Category: flags.AccountCategory, + } + EthRequiredBlocksFlag = &cli.StringFlag{ + Name: "eth.requiredblocks", + Usage: "Comma separated block number-to-hash mappings to require for peering (=)", + Category: flags.EthCategory, + } + LegacyWhitelistFlag = &cli.StringFlag{ + Name: "whitelist", + Usage: "Comma separated block number-to-hash mappings to enforce (=) (deprecated in favor of --eth.requiredblocks)", + Category: flags.DeprecatedCategory, + } + BloomFilterSizeFlag = &cli.Uint64Flag{ + Name: "bloomfilter.size", + Usage: "Megabytes of memory allocated to bloom-filter for pruning", + Value: 2048, + Category: flags.EthCategory, + } + OverrideTerminalTotalDifficulty = &flags.BigFlag{ + Name: "override.terminaltotaldifficulty", + Usage: "Manually specify TerminalTotalDifficulty, overriding the bundled setting", + Category: flags.EthCategory, + } + OverrideTerminalTotalDifficultyPassed = &cli.BoolFlag{ + Name: "override.terminaltotaldifficultypassed", + Usage: "Manually specify TerminalTotalDifficultyPassed, overriding the bundled setting", + Category: flags.EthCategory, } // Light server and client settings - LightServeFlag = cli.IntFlag{ - Name: "light.serve", - Usage: "Maximum percentage of time allowed for serving LES requests (multi-threaded processing allows values over 100)", - Value: ethconfig.Defaults.LightServ, - } - LightIngressFlag = cli.IntFlag{ - Name: "light.ingress", - Usage: "Incoming bandwidth limit for serving light clients (kilobytes/sec, 0 = unlimited)", - Value: ethconfig.Defaults.LightIngress, - } - LightEgressFlag = cli.IntFlag{ - Name: "light.egress", - Usage: "Outgoing bandwidth limit for serving light clients (kilobytes/sec, 0 = unlimited)", - Value: ethconfig.Defaults.LightEgress, - } - LightMaxPeersFlag = cli.IntFlag{ - Name: "light.maxpeers", - Usage: "Maximum number of light clients to serve, or light servers to attach to", - Value: ethconfig.Defaults.LightPeers, - } - UltraLightServersFlag = cli.StringFlag{ - Name: "ulc.servers", - Usage: "List of trusted ultra-light servers", - Value: strings.Join(ethconfig.Defaults.UltraLightServers, ","), - } - UltraLightFractionFlag = cli.IntFlag{ - Name: "ulc.fraction", - Usage: "Minimum % of trusted ultra-light servers required to announce a new head", - Value: ethconfig.Defaults.UltraLightFraction, - } - UltraLightOnlyAnnounceFlag = cli.BoolFlag{ - Name: "ulc.onlyannounce", - Usage: "Ultra light server sends announcements only", - } - LightNoPruneFlag = cli.BoolFlag{ - Name: "light.nopruning", - Usage: "Disable ancient light chain data pruning", - } - LightNoSyncServeFlag = cli.BoolFlag{ - Name: "light.nosyncserve", - Usage: "Enables serving light clients before syncing", + LightServeFlag = &cli.IntFlag{ + Name: "light.serve", + Usage: "Maximum percentage of time allowed for serving LES requests (multi-threaded processing allows values over 100)", + Value: ethconfig.Defaults.LightServ, + Category: flags.LightCategory, + } + LightIngressFlag = &cli.IntFlag{ + Name: "light.ingress", + Usage: "Incoming bandwidth limit for serving light clients (kilobytes/sec, 0 = unlimited)", + Value: ethconfig.Defaults.LightIngress, + Category: flags.LightCategory, + } + LightEgressFlag = &cli.IntFlag{ + Name: "light.egress", + Usage: "Outgoing bandwidth limit for serving light clients (kilobytes/sec, 0 = unlimited)", + Value: ethconfig.Defaults.LightEgress, + Category: flags.LightCategory, + } + LightMaxPeersFlag = &cli.IntFlag{ + Name: "light.maxpeers", + Usage: "Maximum number of light clients to serve, or light servers to attach to", + Value: ethconfig.Defaults.LightPeers, + Category: flags.LightCategory, + } + UltraLightServersFlag = &cli.StringFlag{ + Name: "ulc.servers", + Usage: "List of trusted ultra-light servers", + Value: strings.Join(ethconfig.Defaults.UltraLightServers, ","), + Category: flags.LightCategory, + } + UltraLightFractionFlag = &cli.IntFlag{ + Name: "ulc.fraction", + Usage: "Minimum % of trusted ultra-light servers required to announce a new head", + Value: ethconfig.Defaults.UltraLightFraction, + Category: flags.LightCategory, + } + UltraLightOnlyAnnounceFlag = &cli.BoolFlag{ + Name: "ulc.onlyannounce", + Usage: "Ultra light server sends announcements only", + Category: flags.LightCategory, + } + LightNoPruneFlag = &cli.BoolFlag{ + Name: "light.nopruning", + Usage: "Disable ancient light chain data pruning", + Category: flags.LightCategory, + } + LightNoSyncServeFlag = &cli.BoolFlag{ + Name: "light.nosyncserve", + Usage: "Enables serving light clients before syncing", + Category: flags.LightCategory, } + // Ethash settings - EthashCacheDirFlag = DirectoryFlag{ - Name: "ethash.cachedir", - Usage: "Directory to store the ethash verification caches (default = inside the datadir)", - } - EthashCachesInMemoryFlag = cli.IntFlag{ - Name: "ethash.cachesinmem", - Usage: "Number of recent ethash caches to keep in memory (16MB each)", - Value: ethconfig.Defaults.Ethash.CachesInMem, - } - EthashCachesOnDiskFlag = cli.IntFlag{ - Name: "ethash.cachesondisk", - Usage: "Number of recent ethash caches to keep on disk (16MB each)", - Value: ethconfig.Defaults.Ethash.CachesOnDisk, - } - EthashCachesLockMmapFlag = cli.BoolFlag{ - Name: "ethash.cacheslockmmap", - Usage: "Lock memory maps of recent ethash caches", - } - EthashDatasetDirFlag = DirectoryFlag{ - Name: "ethash.dagdir", - Usage: "Directory to store the ethash mining DAGs", - Value: DirectoryString(ethconfig.Defaults.Ethash.DatasetDir), - } - EthashDatasetsInMemoryFlag = cli.IntFlag{ - Name: "ethash.dagsinmem", - Usage: "Number of recent ethash mining DAGs to keep in memory (1+GB each)", - Value: ethconfig.Defaults.Ethash.DatasetsInMem, - } - EthashDatasetsOnDiskFlag = cli.IntFlag{ - Name: "ethash.dagsondisk", - Usage: "Number of recent ethash mining DAGs to keep on disk (1+GB each)", - Value: ethconfig.Defaults.Ethash.DatasetsOnDisk, - } - EthashDatasetsLockMmapFlag = cli.BoolFlag{ - Name: "ethash.dagslockmmap", - Usage: "Lock memory maps for recent ethash mining DAGs", + EthashCacheDirFlag = &flags.DirectoryFlag{ + Name: "ethash.cachedir", + Usage: "Directory to store the ethash verification caches (default = inside the datadir)", + Category: flags.EthashCategory, + } + EthashCachesInMemoryFlag = &cli.IntFlag{ + Name: "ethash.cachesinmem", + Usage: "Number of recent ethash caches to keep in memory (16MB each)", + Value: ethconfig.Defaults.Ethash.CachesInMem, + Category: flags.EthashCategory, + } + EthashCachesOnDiskFlag = &cli.IntFlag{ + Name: "ethash.cachesondisk", + Usage: "Number of recent ethash caches to keep on disk (16MB each)", + Value: ethconfig.Defaults.Ethash.CachesOnDisk, + Category: flags.EthashCategory, + } + EthashCachesLockMmapFlag = &cli.BoolFlag{ + Name: "ethash.cacheslockmmap", + Usage: "Lock memory maps of recent ethash caches", + Category: flags.EthashCategory, + } + EthashDatasetDirFlag = &flags.DirectoryFlag{ + Name: "ethash.dagdir", + Usage: "Directory to store the ethash mining DAGs", + Value: flags.DirectoryString(ethconfig.Defaults.Ethash.DatasetDir), + Category: flags.EthashCategory, + } + EthashDatasetsInMemoryFlag = &cli.IntFlag{ + Name: "ethash.dagsinmem", + Usage: "Number of recent ethash mining DAGs to keep in memory (1+GB each)", + Value: ethconfig.Defaults.Ethash.DatasetsInMem, + Category: flags.EthashCategory, + } + EthashDatasetsOnDiskFlag = &cli.IntFlag{ + Name: "ethash.dagsondisk", + Usage: "Number of recent ethash mining DAGs to keep on disk (1+GB each)", + Value: ethconfig.Defaults.Ethash.DatasetsOnDisk, + Category: flags.EthashCategory, + } + EthashDatasetsLockMmapFlag = &cli.BoolFlag{ + Name: "ethash.dagslockmmap", + Usage: "Lock memory maps for recent ethash mining DAGs", + Category: flags.EthashCategory, } + // Transaction pool settings - TxPoolLocalsFlag = cli.StringFlag{ - Name: "txpool.locals", - Usage: "Comma separated accounts to treat as locals (no flush, priority inclusion)", - } - TxPoolNoLocalsFlag = cli.BoolFlag{ - Name: "txpool.nolocals", - Usage: "Disables price exemptions for locally submitted transactions", - } - TxPoolJournalFlag = cli.StringFlag{ - Name: "txpool.journal", - Usage: "Disk journal for local transaction to survive node restarts", - Value: core.DefaultTxPoolConfig.Journal, - } - TxPoolRejournalFlag = cli.DurationFlag{ - Name: "txpool.rejournal", - Usage: "Time interval to regenerate the local transaction journal", - Value: core.DefaultTxPoolConfig.Rejournal, - } - TxPoolPriceLimitFlag = cli.Uint64Flag{ - Name: "txpool.pricelimit", - Usage: "Minimum gas price limit to enforce for acceptance into the pool", - Value: ethconfig.Defaults.TxPool.PriceLimit, - } - TxPoolPriceBumpFlag = cli.Uint64Flag{ - Name: "txpool.pricebump", - Usage: "Price bump percentage to replace an already existing transaction", - Value: ethconfig.Defaults.TxPool.PriceBump, - } - TxPoolAccountSlotsFlag = cli.Uint64Flag{ - Name: "txpool.accountslots", - Usage: "Minimum number of executable transaction slots guaranteed per account", - Value: ethconfig.Defaults.TxPool.AccountSlots, - } - TxPoolGlobalSlotsFlag = cli.Uint64Flag{ - Name: "txpool.globalslots", - Usage: "Maximum number of executable transaction slots for all accounts", - Value: ethconfig.Defaults.TxPool.GlobalSlots, - } - TxPoolAccountQueueFlag = cli.Uint64Flag{ - Name: "txpool.accountqueue", - Usage: "Maximum number of non-executable transaction slots permitted per account", - Value: ethconfig.Defaults.TxPool.AccountQueue, - } - TxPoolGlobalQueueFlag = cli.Uint64Flag{ - Name: "txpool.globalqueue", - Usage: "Maximum number of non-executable transaction slots for all accounts", - Value: ethconfig.Defaults.TxPool.GlobalQueue, - } - TxPoolLifetimeFlag = cli.DurationFlag{ - Name: "txpool.lifetime", - Usage: "Maximum amount of time non-executable transaction are queued", - Value: ethconfig.Defaults.TxPool.Lifetime, + TxPoolLocalsFlag = &cli.StringFlag{ + Name: "txpool.locals", + Usage: "Comma separated accounts to treat as locals (no flush, priority inclusion)", + Category: flags.TxPoolCategory, + } + TxPoolNoLocalsFlag = &cli.BoolFlag{ + Name: "txpool.nolocals", + Usage: "Disables price exemptions for locally submitted transactions", + Category: flags.TxPoolCategory, + } + TxPoolJournalFlag = &cli.StringFlag{ + Name: "txpool.journal", + Usage: "Disk journal for local transaction to survive node restarts", + Value: core.DefaultTxPoolConfig.Journal, + Category: flags.TxPoolCategory, + } + TxPoolRejournalFlag = &cli.DurationFlag{ + Name: "txpool.rejournal", + Usage: "Time interval to regenerate the local transaction journal", + Value: core.DefaultTxPoolConfig.Rejournal, + Category: flags.TxPoolCategory, + } + TxPoolPriceLimitFlag = &cli.Uint64Flag{ + Name: "txpool.pricelimit", + Usage: "Minimum gas price limit to enforce for acceptance into the pool", + Value: ethconfig.Defaults.TxPool.PriceLimit, + Category: flags.TxPoolCategory, + } + TxPoolPriceBumpFlag = &cli.Uint64Flag{ + Name: "txpool.pricebump", + Usage: "Price bump percentage to replace an already existing transaction", + Value: ethconfig.Defaults.TxPool.PriceBump, + Category: flags.TxPoolCategory, + } + TxPoolAccountSlotsFlag = &cli.Uint64Flag{ + Name: "txpool.accountslots", + Usage: "Minimum number of executable transaction slots guaranteed per account", + Value: ethconfig.Defaults.TxPool.AccountSlots, + Category: flags.TxPoolCategory, + } + TxPoolGlobalSlotsFlag = &cli.Uint64Flag{ + Name: "txpool.globalslots", + Usage: "Maximum number of executable transaction slots for all accounts", + Value: ethconfig.Defaults.TxPool.GlobalSlots, + Category: flags.TxPoolCategory, + } + TxPoolAccountQueueFlag = &cli.Uint64Flag{ + Name: "txpool.accountqueue", + Usage: "Maximum number of non-executable transaction slots permitted per account", + Value: ethconfig.Defaults.TxPool.AccountQueue, + Category: flags.TxPoolCategory, + } + TxPoolGlobalQueueFlag = &cli.Uint64Flag{ + Name: "txpool.globalqueue", + Usage: "Maximum number of non-executable transaction slots for all accounts", + Value: ethconfig.Defaults.TxPool.GlobalQueue, + Category: flags.TxPoolCategory, + } + TxPoolLifetimeFlag = &cli.DurationFlag{ + Name: "txpool.lifetime", + Usage: "Maximum amount of time non-executable transaction are queued", + Value: ethconfig.Defaults.TxPool.Lifetime, + Category: flags.TxPoolCategory, } + // Performance tuning settings - CacheFlag = cli.IntFlag{ - Name: "cache", - Usage: "Megabytes of memory allocated to internal caching (default = 4096 mainnet full node, 128 light mode)", - Value: 1024, - } - CacheDatabaseFlag = cli.IntFlag{ - Name: "cache.database", - Usage: "Percentage of cache memory allowance to use for database io", - Value: 50, - } - CacheTrieFlag = cli.IntFlag{ - Name: "cache.trie", - Usage: "Percentage of cache memory allowance to use for trie caching (default = 15% full mode, 30% archive mode)", - Value: 15, - } - CacheTrieJournalFlag = cli.StringFlag{ - Name: "cache.trie.journal", - Usage: "Disk journal directory for trie cache to survive node restarts", - Value: ethconfig.Defaults.TrieCleanCacheJournal, - } - CacheTrieRejournalFlag = cli.DurationFlag{ - Name: "cache.trie.rejournal", - Usage: "Time interval to regenerate the trie cache journal", - Value: ethconfig.Defaults.TrieCleanCacheRejournal, - } - CacheGCFlag = cli.IntFlag{ - Name: "cache.gc", - Usage: "Percentage of cache memory allowance to use for trie pruning (default = 25% full mode, 0% archive mode)", - Value: 25, - } - CacheSnapshotFlag = cli.IntFlag{ - Name: "cache.snapshot", - Usage: "Percentage of cache memory allowance to use for snapshot caching (default = 10% full mode, 20% archive mode)", - Value: 10, - } - CacheNoPrefetchFlag = cli.BoolFlag{ - Name: "cache.noprefetch", - Usage: "Disable heuristic state prefetch during block import (less CPU and disk IO, more time waiting for data)", - } - CachePreimagesFlag = cli.BoolFlag{ - Name: "cache.preimages", - Usage: "Enable recording the SHA3/keccak preimages of trie keys", - } - FDLimitFlag = cli.IntFlag{ - Name: "fdlimit", - Usage: "Raise the open file descriptor resource limit (default = system fd limit)", + CacheFlag = &cli.IntFlag{ + Name: "cache", + Usage: "Megabytes of memory allocated to internal caching (default = 4096 mainnet full node, 128 light mode)", + Value: 1024, + Category: flags.PerfCategory, + } + CacheDatabaseFlag = &cli.IntFlag{ + Name: "cache.database", + Usage: "Percentage of cache memory allowance to use for database io", + Value: 50, + Category: flags.PerfCategory, + } + CacheTrieFlag = &cli.IntFlag{ + Name: "cache.trie", + Usage: "Percentage of cache memory allowance to use for trie caching (default = 15% full mode, 30% archive mode)", + Value: 15, + Category: flags.PerfCategory, + } + CacheTrieJournalFlag = &cli.StringFlag{ + Name: "cache.trie.journal", + Usage: "Disk journal directory for trie cache to survive node restarts", + Value: ethconfig.Defaults.TrieCleanCacheJournal, + Category: flags.PerfCategory, + } + CacheTrieRejournalFlag = &cli.DurationFlag{ + Name: "cache.trie.rejournal", + Usage: "Time interval to regenerate the trie cache journal", + Value: ethconfig.Defaults.TrieCleanCacheRejournal, + Category: flags.PerfCategory, + } + CacheGCFlag = &cli.IntFlag{ + Name: "cache.gc", + Usage: "Percentage of cache memory allowance to use for trie pruning (default = 25% full mode, 0% archive mode)", + Value: 25, + Category: flags.PerfCategory, + } + CacheSnapshotFlag = &cli.IntFlag{ + Name: "cache.snapshot", + Usage: "Percentage of cache memory allowance to use for snapshot caching (default = 10% full mode, 20% archive mode)", + Value: 10, + Category: flags.PerfCategory, + } + CacheNoPrefetchFlag = &cli.BoolFlag{ + Name: "cache.noprefetch", + Usage: "Disable heuristic state prefetch during block import (less CPU and disk IO, more time waiting for data)", + Category: flags.PerfCategory, + } + CachePreimagesFlag = &cli.BoolFlag{ + Name: "cache.preimages", + Usage: "Enable recording the SHA3/keccak preimages of trie keys", + Category: flags.PerfCategory, + } + CacheLogSizeFlag = &cli.IntFlag{ + Name: "cache.blocklogs", + Usage: "Size (in number of blocks) of the log cache for filtering", + Category: flags.PerfCategory, + Value: ethconfig.Defaults.FilterLogCacheSize, + } + FDLimitFlag = &cli.IntFlag{ + Name: "fdlimit", + Usage: "Raise the open file descriptor resource limit (default = system fd limit)", + Category: flags.PerfCategory, } + // Miner settings - MiningEnabledFlag = cli.BoolFlag{ - Name: "mine", - Usage: "Enable mining", - } - MinerThreadsFlag = cli.IntFlag{ - Name: "miner.threads", - Usage: "Number of CPU threads to use for mining", - Value: 0, - } - MinerNotifyFlag = cli.StringFlag{ - Name: "miner.notify", - Usage: "Comma separated HTTP URL list to notify of new work packages", - } - MinerNotifyFullFlag = cli.BoolFlag{ - Name: "miner.notify.full", - Usage: "Notify with pending block headers instead of work packages", - } - MinerGasLimitFlag = cli.Uint64Flag{ - Name: "miner.gaslimit", - Usage: "Target gas ceiling for mined blocks", - Value: ethconfig.Defaults.Miner.GasCeil, - } - MinerGasPriceFlag = BigFlag{ - Name: "miner.gasprice", - Usage: "Minimum gas price for mining a transaction", - Value: ethconfig.Defaults.Miner.GasPrice, - } - MinerEtherbaseFlag = cli.StringFlag{ - Name: "miner.etherbase", - Usage: "Public address for block mining rewards (default = first account)", - Value: "0", - } - MinerExtraDataFlag = cli.StringFlag{ - Name: "miner.extradata", - Usage: "Block extra data set by the miner (default = client version)", - } - MinerRecommitIntervalFlag = cli.DurationFlag{ - Name: "miner.recommit", - Usage: "Time interval to recreate the block being mined", - Value: ethconfig.Defaults.Miner.Recommit, - } - MinerNoVerifyFlag = cli.BoolFlag{ - Name: "miner.noverify", - Usage: "Disable remote sealing verification", + MiningEnabledFlag = &cli.BoolFlag{ + Name: "mine", + Usage: "Enable mining", + Category: flags.MinerCategory, + } + MinerThreadsFlag = &cli.IntFlag{ + Name: "miner.threads", + Usage: "Number of CPU threads to use for mining", + Value: 0, + Category: flags.MinerCategory, + } + MinerNotifyFlag = &cli.StringFlag{ + Name: "miner.notify", + Usage: "Comma separated HTTP URL list to notify of new work packages", + Category: flags.MinerCategory, + } + MinerNotifyFullFlag = &cli.BoolFlag{ + Name: "miner.notify.full", + Usage: "Notify with pending block headers instead of work packages", + Category: flags.MinerCategory, + } + MinerGasLimitFlag = &cli.Uint64Flag{ + Name: "miner.gaslimit", + Usage: "Target gas ceiling for mined blocks", + Value: ethconfig.Defaults.Miner.GasCeil, + Category: flags.MinerCategory, + } + MinerGasPriceFlag = &flags.BigFlag{ + Name: "miner.gasprice", + Usage: "Minimum gas price for mining a transaction", + Value: ethconfig.Defaults.Miner.GasPrice, + Category: flags.MinerCategory, + } + MinerEtherbaseFlag = &cli.StringFlag{ + Name: "miner.etherbase", + Usage: "Public address for block mining rewards (default = first account)", + Value: "0", + Category: flags.MinerCategory, + } + MinerExtraDataFlag = &cli.StringFlag{ + Name: "miner.extradata", + Usage: "Block extra data set by the miner (default = client version)", + Category: flags.MinerCategory, + } + MinerRecommitIntervalFlag = &cli.DurationFlag{ + Name: "miner.recommit", + Usage: "Time interval to recreate the block being mined", + Value: ethconfig.Defaults.Miner.Recommit, + Category: flags.MinerCategory, + } + MinerNoVerifyFlag = &cli.BoolFlag{ + Name: "miner.noverify", + Usage: "Disable remote sealing verification", + Category: flags.MinerCategory, } + // Account settings - UnlockedAccountFlag = cli.StringFlag{ - Name: "unlock", - Usage: "Comma separated list of accounts to unlock", - Value: "", - } - PasswordFileFlag = cli.StringFlag{ - Name: "password", - Usage: "Password file to use for non-interactive password input", - Value: "", - } - ExternalSignerFlag = cli.StringFlag{ - Name: "signer", - Usage: "External signer (url or path to ipc file)", - Value: "", - } - VMEnableDebugFlag = cli.BoolFlag{ - Name: "vmdebug", - Usage: "Record information useful for VM and contract debugging", - } - InsecureUnlockAllowedFlag = cli.BoolFlag{ - Name: "allow-insecure-unlock", - Usage: "Allow insecure account unlocking when account-related RPCs are exposed by http", - } - RPCGlobalGasCapFlag = cli.Uint64Flag{ - Name: "rpc.gascap", - Usage: "Sets a cap on gas that can be used in eth_call/estimateGas (0=infinite)", - Value: ethconfig.Defaults.RPCGasCap, - } - RPCGlobalEVMTimeoutFlag = cli.DurationFlag{ - Name: "rpc.evmtimeout", - Usage: "Sets a timeout used for eth_call (0=infinite)", - Value: ethconfig.Defaults.RPCEVMTimeout, - } - RPCGlobalTxFeeCapFlag = cli.Float64Flag{ - Name: "rpc.txfeecap", - Usage: "Sets a cap on transaction fee (in ether) that can be sent via the RPC APIs (0 = no cap)", - Value: ethconfig.Defaults.RPCTxFeeCap, + UnlockedAccountFlag = &cli.StringFlag{ + Name: "unlock", + Usage: "Comma separated list of accounts to unlock", + Value: "", + Category: flags.AccountCategory, + } + PasswordFileFlag = &cli.PathFlag{ + Name: "password", + Usage: "Password file to use for non-interactive password input", + TakesFile: true, + Category: flags.AccountCategory, + } + ExternalSignerFlag = &cli.StringFlag{ + Name: "signer", + Usage: "External signer (url or path to ipc file)", + Value: "", + Category: flags.AccountCategory, + } + InsecureUnlockAllowedFlag = &cli.BoolFlag{ + Name: "allow-insecure-unlock", + Usage: "Allow insecure account unlocking when account-related RPCs are exposed by http", + Category: flags.AccountCategory, + } + + // EVM settings + VMEnableDebugFlag = &cli.BoolFlag{ + Name: "vmdebug", + Usage: "Record information useful for VM and contract debugging", + Category: flags.VMCategory, + } + + // API options. + RPCGlobalGasCapFlag = &cli.Uint64Flag{ + Name: "rpc.gascap", + Usage: "Sets a cap on gas that can be used in eth_call/estimateGas (0=infinite)", + Value: ethconfig.Defaults.RPCGasCap, + Category: flags.APICategory, + } + RPCGlobalEVMTimeoutFlag = &cli.DurationFlag{ + Name: "rpc.evmtimeout", + Usage: "Sets a timeout used for eth_call (0=infinite)", + Value: ethconfig.Defaults.RPCEVMTimeout, + Category: flags.APICategory, + } + RPCGlobalTxFeeCapFlag = &cli.Float64Flag{ + Name: "rpc.txfeecap", + Usage: "Sets a cap on transaction fee (in ether) that can be sent via the RPC APIs (0 = no cap)", + Value: ethconfig.Defaults.RPCTxFeeCap, + Category: flags.APICategory, } // Authenticated RPC HTTP settings - AuthListenFlag = cli.StringFlag{ - Name: "authrpc.addr", - Usage: "Listening address for authenticated APIs", - Value: node.DefaultConfig.AuthAddr, - } - AuthPortFlag = cli.IntFlag{ - Name: "authrpc.port", - Usage: "Listening port for authenticated APIs", - Value: node.DefaultConfig.AuthPort, - } - AuthVirtualHostsFlag = cli.StringFlag{ - Name: "authrpc.vhosts", - Usage: "Comma separated list of virtual hostnames from which to accept requests (server enforced). Accepts '*' wildcard.", - Value: strings.Join(node.DefaultConfig.AuthVirtualHosts, ","), - } - JWTSecretFlag = cli.StringFlag{ - Name: "authrpc.jwtsecret", - Usage: "Path to a JWT secret to use for authenticated RPC endpoints", + AuthListenFlag = &cli.StringFlag{ + Name: "authrpc.addr", + Usage: "Listening address for authenticated APIs", + Value: node.DefaultConfig.AuthAddr, + Category: flags.APICategory, + } + AuthPortFlag = &cli.IntFlag{ + Name: "authrpc.port", + Usage: "Listening port for authenticated APIs", + Value: node.DefaultConfig.AuthPort, + Category: flags.APICategory, + } + AuthVirtualHostsFlag = &cli.StringFlag{ + Name: "authrpc.vhosts", + Usage: "Comma separated list of virtual hostnames from which to accept requests (server enforced). Accepts '*' wildcard.", + Value: strings.Join(node.DefaultConfig.AuthVirtualHosts, ","), + Category: flags.APICategory, + } + JWTSecretFlag = &flags.DirectoryFlag{ + Name: "authrpc.jwtsecret", + Usage: "Path to a JWT secret to use for authenticated RPC endpoints", + Category: flags.APICategory, } + // Logging and debug settings - EthStatsURLFlag = cli.StringFlag{ - Name: "ethstats", - Usage: "Reporting URL of a ethstats service (nodename:secret@host:port)", + EthStatsURLFlag = &cli.StringFlag{ + Name: "ethstats", + Usage: "Reporting URL of a ethstats service (nodename:secret@host:port)", + Category: flags.MetricsCategory, } - FakePoWFlag = cli.BoolFlag{ - Name: "fakepow", - Usage: "Disables proof-of-work verification", + FakePoWFlag = &cli.BoolFlag{ + Name: "fakepow", + Usage: "Disables proof-of-work verification", + Category: flags.LoggingCategory, } - NoCompactionFlag = cli.BoolFlag{ - Name: "nocompaction", - Usage: "Disables db compaction after import", + NoCompactionFlag = &cli.BoolFlag{ + Name: "nocompaction", + Usage: "Disables db compaction after import", + Category: flags.LoggingCategory, } - IgnoreLegacyReceiptsFlag = cli.BoolFlag{ - Name: "ignore-legacy-receipts", - Usage: "Geth will start up even if there are legacy receipts in freezer", + + IgnoreLegacyReceiptsFlag = &cli.BoolFlag{ + Name: "ignore-legacy-receipts", + Usage: "Geth will start up even if there are legacy receipts in freezer", + Category: flags.MiscCategory, } + // RPC settings - IPCDisabledFlag = cli.BoolFlag{ - Name: "ipcdisable", - Usage: "Disable the IPC-RPC server", - } - IPCPathFlag = DirectoryFlag{ - Name: "ipcpath", - Usage: "Filename for IPC socket/pipe within the datadir (explicit paths escape it)", - } - HTTPEnabledFlag = cli.BoolFlag{ - Name: "http", - Usage: "Enable the HTTP-RPC server", - } - HTTPListenAddrFlag = cli.StringFlag{ - Name: "http.addr", - Usage: "HTTP-RPC server listening interface", - Value: node.DefaultHTTPHost, - } - HTTPPortFlag = cli.IntFlag{ - Name: "http.port", - Usage: "HTTP-RPC server listening port", - Value: node.DefaultHTTPPort, - } - HTTPCORSDomainFlag = cli.StringFlag{ - Name: "http.corsdomain", - Usage: "Comma separated list of domains from which to accept cross origin requests (browser enforced)", - Value: "", - } - HTTPVirtualHostsFlag = cli.StringFlag{ - Name: "http.vhosts", - Usage: "Comma separated list of virtual hostnames from which to accept requests (server enforced). Accepts '*' wildcard.", - Value: strings.Join(node.DefaultConfig.HTTPVirtualHosts, ","), - } - HTTPApiFlag = cli.StringFlag{ - Name: "http.api", - Usage: "API's offered over the HTTP-RPC interface", - Value: "", - } - HTTPPathPrefixFlag = cli.StringFlag{ - Name: "http.rpcprefix", - Usage: "HTTP path path prefix on which JSON-RPC is served. Use '/' to serve on all paths.", - Value: "", - } - GraphQLEnabledFlag = cli.BoolFlag{ - Name: "graphql", - Usage: "Enable GraphQL on the HTTP-RPC server. Note that GraphQL can only be started if an HTTP server is started as well.", - } - GraphQLCORSDomainFlag = cli.StringFlag{ - Name: "graphql.corsdomain", - Usage: "Comma separated list of domains from which to accept cross origin requests (browser enforced)", - Value: "", - } - GraphQLVirtualHostsFlag = cli.StringFlag{ - Name: "graphql.vhosts", - Usage: "Comma separated list of virtual hostnames from which to accept requests (server enforced). Accepts '*' wildcard.", - Value: strings.Join(node.DefaultConfig.GraphQLVirtualHosts, ","), - } - WSEnabledFlag = cli.BoolFlag{ - Name: "ws", - Usage: "Enable the WS-RPC server", - } - WSListenAddrFlag = cli.StringFlag{ - Name: "ws.addr", - Usage: "WS-RPC server listening interface", - Value: node.DefaultWSHost, - } - WSPortFlag = cli.IntFlag{ - Name: "ws.port", - Usage: "WS-RPC server listening port", - Value: node.DefaultWSPort, - } - WSApiFlag = cli.StringFlag{ - Name: "ws.api", - Usage: "API's offered over the WS-RPC interface", - Value: "", - } - WSAllowedOriginsFlag = cli.StringFlag{ - Name: "ws.origins", - Usage: "Origins from which to accept websockets requests", - Value: "", - } - WSPathPrefixFlag = cli.StringFlag{ - Name: "ws.rpcprefix", - Usage: "HTTP path prefix on which JSON-RPC is served. Use '/' to serve on all paths.", - Value: "", - } - ExecFlag = cli.StringFlag{ - Name: "exec", - Usage: "Execute JavaScript statement", - } - PreloadJSFlag = cli.StringFlag{ - Name: "preload", - Usage: "Comma separated list of JavaScript files to preload into the console", - } - AllowUnprotectedTxs = cli.BoolFlag{ - Name: "rpc.allow-unprotected-txs", - Usage: "Allow for unprotected (non EIP155 signed) transactions to be submitted via RPC", + IPCDisabledFlag = &cli.BoolFlag{ + Name: "ipcdisable", + Usage: "Disable the IPC-RPC server", + Category: flags.APICategory, + } + IPCPathFlag = &flags.DirectoryFlag{ + Name: "ipcpath", + Usage: "Filename for IPC socket/pipe within the datadir (explicit paths escape it)", + Category: flags.APICategory, + } + HTTPEnabledFlag = &cli.BoolFlag{ + Name: "http", + Usage: "Enable the HTTP-RPC server", + Category: flags.APICategory, + } + HTTPListenAddrFlag = &cli.StringFlag{ + Name: "http.addr", + Usage: "HTTP-RPC server listening interface", + Value: node.DefaultHTTPHost, + Category: flags.APICategory, + } + HTTPPortFlag = &cli.IntFlag{ + Name: "http.port", + Usage: "HTTP-RPC server listening port", + Value: node.DefaultHTTPPort, + Category: flags.APICategory, + } + HTTPCORSDomainFlag = &cli.StringFlag{ + Name: "http.corsdomain", + Usage: "Comma separated list of domains from which to accept cross origin requests (browser enforced)", + Value: "", + Category: flags.APICategory, + } + HTTPVirtualHostsFlag = &cli.StringFlag{ + Name: "http.vhosts", + Usage: "Comma separated list of virtual hostnames from which to accept requests (server enforced). Accepts '*' wildcard.", + Value: strings.Join(node.DefaultConfig.HTTPVirtualHosts, ","), + Category: flags.APICategory, + } + HTTPApiFlag = &cli.StringFlag{ + Name: "http.api", + Usage: "API's offered over the HTTP-RPC interface", + Value: "", + Category: flags.APICategory, + } + HTTPPathPrefixFlag = &cli.StringFlag{ + Name: "http.rpcprefix", + Usage: "HTTP path path prefix on which JSON-RPC is served. Use '/' to serve on all paths.", + Value: "", + Category: flags.APICategory, + } + GraphQLEnabledFlag = &cli.BoolFlag{ + Name: "graphql", + Usage: "Enable GraphQL on the HTTP-RPC server. Note that GraphQL can only be started if an HTTP server is started as well.", + Category: flags.APICategory, + } + GraphQLCORSDomainFlag = &cli.StringFlag{ + Name: "graphql.corsdomain", + Usage: "Comma separated list of domains from which to accept cross origin requests (browser enforced)", + Value: "", + Category: flags.APICategory, + } + GraphQLVirtualHostsFlag = &cli.StringFlag{ + Name: "graphql.vhosts", + Usage: "Comma separated list of virtual hostnames from which to accept requests (server enforced). Accepts '*' wildcard.", + Value: strings.Join(node.DefaultConfig.GraphQLVirtualHosts, ","), + Category: flags.APICategory, + } + WSEnabledFlag = &cli.BoolFlag{ + Name: "ws", + Usage: "Enable the WS-RPC server", + Category: flags.APICategory, + } + WSListenAddrFlag = &cli.StringFlag{ + Name: "ws.addr", + Usage: "WS-RPC server listening interface", + Value: node.DefaultWSHost, + Category: flags.APICategory, + } + WSPortFlag = &cli.IntFlag{ + Name: "ws.port", + Usage: "WS-RPC server listening port", + Value: node.DefaultWSPort, + Category: flags.APICategory, + } + WSApiFlag = &cli.StringFlag{ + Name: "ws.api", + Usage: "API's offered over the WS-RPC interface", + Value: "", + Category: flags.APICategory, + } + WSAllowedOriginsFlag = &cli.StringFlag{ + Name: "ws.origins", + Usage: "Origins from which to accept websockets requests", + Value: "", + Category: flags.APICategory, + } + WSPathPrefixFlag = &cli.StringFlag{ + Name: "ws.rpcprefix", + Usage: "HTTP path prefix on which JSON-RPC is served. Use '/' to serve on all paths.", + Value: "", + Category: flags.APICategory, + } + ExecFlag = &cli.StringFlag{ + Name: "exec", + Usage: "Execute JavaScript statement", + Category: flags.APICategory, + } + PreloadJSFlag = &cli.StringFlag{ + Name: "preload", + Usage: "Comma separated list of JavaScript files to preload into the console", + Category: flags.APICategory, + } + AllowUnprotectedTxs = &cli.BoolFlag{ + Name: "rpc.allow-unprotected-txs", + Usage: "Allow for unprotected (non EIP155 signed) transactions to be submitted via RPC", + Category: flags.APICategory, } // Network Settings - MaxPeersFlag = cli.IntFlag{ - Name: "maxpeers", - Usage: "Maximum number of network peers (network disabled if set to 0)", - Value: node.DefaultConfig.P2P.MaxPeers, - } - MaxPendingPeersFlag = cli.IntFlag{ - Name: "maxpendpeers", - Usage: "Maximum number of pending connection attempts (defaults used if set to 0)", - Value: node.DefaultConfig.P2P.MaxPendingPeers, - } - ListenPortFlag = cli.IntFlag{ - Name: "port", - Usage: "Network listening port", - Value: 30303, - } - BootnodesFlag = cli.StringFlag{ - Name: "bootnodes", - Usage: "Comma separated enode URLs for P2P discovery bootstrap", - Value: "", - } - NodeKeyFileFlag = cli.StringFlag{ - Name: "nodekey", - Usage: "P2P node key file", - } - NodeKeyHexFlag = cli.StringFlag{ - Name: "nodekeyhex", - Usage: "P2P node key as hex (for testing)", - } - NATFlag = cli.StringFlag{ - Name: "nat", - Usage: "NAT port mapping mechanism (any|none|upnp|pmp|extip:)", - Value: "any", - } - NoDiscoverFlag = cli.BoolFlag{ - Name: "nodiscover", - Usage: "Disables the peer discovery mechanism (manual peer addition)", - } - DiscoveryV5Flag = cli.BoolFlag{ - Name: "v5disc", - Usage: "Enables the experimental RLPx V5 (Topic Discovery) mechanism", - } - NetrestrictFlag = cli.StringFlag{ - Name: "netrestrict", - Usage: "Restricts network communication to the given IP networks (CIDR masks)", - } - DNSDiscoveryFlag = cli.StringFlag{ - Name: "discovery.dns", - Usage: "Sets DNS discovery entry points (use \"\" to disable DNS)", - } - - // ATM the url is left to the user and deployment to - JSpathFlag = DirectoryFlag{ - Name: "jspath", - Usage: "JavaScript root path for `loadScript`", - Value: DirectoryString("."), + MaxPeersFlag = &cli.IntFlag{ + Name: "maxpeers", + Usage: "Maximum number of network peers (network disabled if set to 0)", + Value: node.DefaultConfig.P2P.MaxPeers, + Category: flags.NetworkingCategory, + } + MaxPendingPeersFlag = &cli.IntFlag{ + Name: "maxpendpeers", + Usage: "Maximum number of pending connection attempts (defaults used if set to 0)", + Value: node.DefaultConfig.P2P.MaxPendingPeers, + Category: flags.NetworkingCategory, + } + ListenPortFlag = &cli.IntFlag{ + Name: "port", + Usage: "Network listening port", + Value: 30303, + Category: flags.NetworkingCategory, + } + BootnodesFlag = &cli.StringFlag{ + Name: "bootnodes", + Usage: "Comma separated enode URLs for P2P discovery bootstrap", + Value: "", + Category: flags.NetworkingCategory, + } + NodeKeyFileFlag = &cli.StringFlag{ + Name: "nodekey", + Usage: "P2P node key file", + Category: flags.NetworkingCategory, + } + NodeKeyHexFlag = &cli.StringFlag{ + Name: "nodekeyhex", + Usage: "P2P node key as hex (for testing)", + Category: flags.NetworkingCategory, + } + NATFlag = &cli.StringFlag{ + Name: "nat", + Usage: "NAT port mapping mechanism (any|none|upnp|pmp|extip:)", + Value: "any", + Category: flags.NetworkingCategory, + } + NoDiscoverFlag = &cli.BoolFlag{ + Name: "nodiscover", + Usage: "Disables the peer discovery mechanism (manual peer addition)", + Category: flags.NetworkingCategory, + } + DiscoveryV5Flag = &cli.BoolFlag{ + Name: "v5disc", + Usage: "Enables the experimental RLPx V5 (Topic Discovery) mechanism", + Category: flags.NetworkingCategory, + } + NetrestrictFlag = &cli.StringFlag{ + Name: "netrestrict", + Usage: "Restricts network communication to the given IP networks (CIDR masks)", + Category: flags.NetworkingCategory, + } + DNSDiscoveryFlag = &cli.StringFlag{ + Name: "discovery.dns", + Usage: "Sets DNS discovery entry points (use \"\" to disable DNS)", + Category: flags.NetworkingCategory, + } + DiscoveryPortFlag = &cli.IntFlag{ + Name: "discovery.port", + Usage: "Use a custom UDP port for P2P discovery", + Value: 30303, + Category: flags.NetworkingCategory, + } + + // Console + JSpathFlag = &flags.DirectoryFlag{ + Name: "jspath", + Usage: "JavaScript root path for `loadScript`", + Value: flags.DirectoryString("."), + Category: flags.APICategory, } // Gas price oracle settings - GpoBlocksFlag = cli.IntFlag{ - Name: "gpo.blocks", - Usage: "Number of recent blocks to check for gas prices", - Value: ethconfig.Defaults.GPO.Blocks, - } - GpoPercentileFlag = cli.IntFlag{ - Name: "gpo.percentile", - Usage: "Suggested gas price is the given percentile of a set of recent transaction gas prices", - Value: ethconfig.Defaults.GPO.Percentile, - } - GpoMaxGasPriceFlag = cli.Int64Flag{ - Name: "gpo.maxprice", - Usage: "Maximum transaction priority fee (or gasprice before London fork) to be recommended by gpo", - Value: ethconfig.Defaults.GPO.MaxPrice.Int64(), - } - GpoIgnoreGasPriceFlag = cli.Int64Flag{ - Name: "gpo.ignoreprice", - Usage: "Gas price below which gpo will ignore transactions", - Value: ethconfig.Defaults.GPO.IgnorePrice.Int64(), + GpoBlocksFlag = &cli.IntFlag{ + Name: "gpo.blocks", + Usage: "Number of recent blocks to check for gas prices", + Value: ethconfig.Defaults.GPO.Blocks, + Category: flags.GasPriceCategory, + } + GpoPercentileFlag = &cli.IntFlag{ + Name: "gpo.percentile", + Usage: "Suggested gas price is the given percentile of a set of recent transaction gas prices", + Value: ethconfig.Defaults.GPO.Percentile, + Category: flags.GasPriceCategory, + } + GpoMaxGasPriceFlag = &cli.Int64Flag{ + Name: "gpo.maxprice", + Usage: "Maximum transaction priority fee (or gasprice before London fork) to be recommended by gpo", + Value: ethconfig.Defaults.GPO.MaxPrice.Int64(), + Category: flags.GasPriceCategory, + } + GpoIgnoreGasPriceFlag = &cli.Int64Flag{ + Name: "gpo.ignoreprice", + Usage: "Gas price below which gpo will ignore transactions", + Value: ethconfig.Defaults.GPO.IgnorePrice.Int64(), + Category: flags.GasPriceCategory, } // Metrics flags - MetricsEnabledFlag = cli.BoolFlag{ - Name: "metrics", - Usage: "Enable metrics collection and reporting", + MetricsEnabledFlag = &cli.BoolFlag{ + Name: "metrics", + Usage: "Enable metrics collection and reporting", + Category: flags.MetricsCategory, } - MetricsEnabledExpensiveFlag = cli.BoolFlag{ - Name: "metrics.expensive", - Usage: "Enable expensive metrics collection and reporting", + MetricsEnabledExpensiveFlag = &cli.BoolFlag{ + Name: "metrics.expensive", + Usage: "Enable expensive metrics collection and reporting", + Category: flags.MetricsCategory, } // MetricsHTTPFlag defines the endpoint for a stand-alone metrics HTTP endpoint. // Since the pprof service enables sensitive/vulnerable behavior, this allows a user // to enable a public-OK metrics endpoint without having to worry about ALSO exposing // other profiling behavior or information. - MetricsHTTPFlag = cli.StringFlag{ - Name: "metrics.addr", - Usage: "Enable stand-alone metrics HTTP server listening interface", - Value: metrics.DefaultConfig.HTTP, - } - MetricsPortFlag = cli.IntFlag{ - Name: "metrics.port", - Usage: "Metrics HTTP server listening port", - Value: metrics.DefaultConfig.Port, - } - MetricsEnableInfluxDBFlag = cli.BoolFlag{ - Name: "metrics.influxdb", - Usage: "Enable metrics export/push to an external InfluxDB database", - } - MetricsInfluxDBEndpointFlag = cli.StringFlag{ - Name: "metrics.influxdb.endpoint", - Usage: "InfluxDB API endpoint to report metrics to", - Value: metrics.DefaultConfig.InfluxDBEndpoint, - } - MetricsInfluxDBDatabaseFlag = cli.StringFlag{ - Name: "metrics.influxdb.database", - Usage: "InfluxDB database name to push reported metrics to", - Value: metrics.DefaultConfig.InfluxDBDatabase, - } - MetricsInfluxDBUsernameFlag = cli.StringFlag{ - Name: "metrics.influxdb.username", - Usage: "Username to authorize access to the database", - Value: metrics.DefaultConfig.InfluxDBUsername, - } - MetricsInfluxDBPasswordFlag = cli.StringFlag{ - Name: "metrics.influxdb.password", - Usage: "Password to authorize access to the database", - Value: metrics.DefaultConfig.InfluxDBPassword, + MetricsHTTPFlag = &cli.StringFlag{ + Name: "metrics.addr", + Usage: "Enable stand-alone metrics HTTP server listening interface", + Value: metrics.DefaultConfig.HTTP, + Category: flags.MetricsCategory, + } + MetricsPortFlag = &cli.IntFlag{ + Name: "metrics.port", + Usage: "Metrics HTTP server listening port", + Value: metrics.DefaultConfig.Port, + Category: flags.MetricsCategory, + } + MetricsEnableInfluxDBFlag = &cli.BoolFlag{ + Name: "metrics.influxdb", + Usage: "Enable metrics export/push to an external InfluxDB database", + Category: flags.MetricsCategory, + } + MetricsInfluxDBEndpointFlag = &cli.StringFlag{ + Name: "metrics.influxdb.endpoint", + Usage: "InfluxDB API endpoint to report metrics to", + Value: metrics.DefaultConfig.InfluxDBEndpoint, + Category: flags.MetricsCategory, + } + MetricsInfluxDBDatabaseFlag = &cli.StringFlag{ + Name: "metrics.influxdb.database", + Usage: "InfluxDB database name to push reported metrics to", + Value: metrics.DefaultConfig.InfluxDBDatabase, + Category: flags.MetricsCategory, + } + MetricsInfluxDBUsernameFlag = &cli.StringFlag{ + Name: "metrics.influxdb.username", + Usage: "Username to authorize access to the database", + Value: metrics.DefaultConfig.InfluxDBUsername, + Category: flags.MetricsCategory, + } + MetricsInfluxDBPasswordFlag = &cli.StringFlag{ + Name: "metrics.influxdb.password", + Usage: "Password to authorize access to the database", + Value: metrics.DefaultConfig.InfluxDBPassword, + Category: flags.MetricsCategory, } // Tags are part of every measurement sent to InfluxDB. Queries on tags are faster in InfluxDB. // For example `host` tag could be used so that we can group all nodes and average a measurement // across all of them, but also so that we can select a specific node and inspect its measurements. // https://docs.influxdata.com/influxdb/v1.4/concepts/key_concepts/#tag-key - MetricsInfluxDBTagsFlag = cli.StringFlag{ - Name: "metrics.influxdb.tags", - Usage: "Comma-separated InfluxDB tags (key/values) attached to all measurements", - Value: metrics.DefaultConfig.InfluxDBTags, + MetricsInfluxDBTagsFlag = &cli.StringFlag{ + Name: "metrics.influxdb.tags", + Usage: "Comma-separated InfluxDB tags (key/values) attached to all measurements", + Value: metrics.DefaultConfig.InfluxDBTags, + Category: flags.MetricsCategory, } - MetricsEnableInfluxDBV2Flag = cli.BoolFlag{ - Name: "metrics.influxdbv2", - Usage: "Enable metrics export/push to an external InfluxDB v2 database", + MetricsEnableInfluxDBV2Flag = &cli.BoolFlag{ + Name: "metrics.influxdbv2", + Usage: "Enable metrics export/push to an external InfluxDB v2 database", + Category: flags.MetricsCategory, } - MetricsInfluxDBTokenFlag = cli.StringFlag{ - Name: "metrics.influxdb.token", - Usage: "Token to authorize access to the database (v2 only)", - Value: metrics.DefaultConfig.InfluxDBToken, + MetricsInfluxDBTokenFlag = &cli.StringFlag{ + Name: "metrics.influxdb.token", + Usage: "Token to authorize access to the database (v2 only)", + Value: metrics.DefaultConfig.InfluxDBToken, + Category: flags.MetricsCategory, } - MetricsInfluxDBBucketFlag = cli.StringFlag{ - Name: "metrics.influxdb.bucket", - Usage: "InfluxDB bucket name to push reported metrics to (v2 only)", - Value: metrics.DefaultConfig.InfluxDBBucket, + MetricsInfluxDBBucketFlag = &cli.StringFlag{ + Name: "metrics.influxdb.bucket", + Usage: "InfluxDB bucket name to push reported metrics to (v2 only)", + Value: metrics.DefaultConfig.InfluxDBBucket, + Category: flags.MetricsCategory, } - MetricsInfluxDBOrganizationFlag = cli.StringFlag{ - Name: "metrics.influxdb.organization", - Usage: "InfluxDB organization name (v2 only)", - Value: metrics.DefaultConfig.InfluxDBOrganization, + MetricsInfluxDBOrganizationFlag = &cli.StringFlag{ + Name: "metrics.influxdb.organization", + Usage: "InfluxDB organization name (v2 only)", + Value: metrics.DefaultConfig.InfluxDBOrganization, + Category: flags.MetricsCategory, + } + + HttpHeaderFlag = &cli.StringSliceFlag{ + Name: "header", + Aliases: []string{"H"}, + Usage: "Pass custom headers to the RPC server wheng using --" + RemoteDBFlag.Name + " or the geth attach console.", + Category: flags.NetworkingCategory, } ) @@ -841,47 +998,37 @@ var ( KilnFlag, } // NetworkFlags is the flag group of all built-in supported networks. - NetworkFlags = append([]cli.Flag{ - MainnetFlag, - }, TestnetFlags...) + NetworkFlags = append([]cli.Flag{MainnetFlag}, TestnetFlags...) // DatabasePathFlags is the flag group of all database path flags. DatabasePathFlags = []cli.Flag{ DataDirFlag, AncientFlag, RemoteDBFlag, + HttpHeaderFlag, } ) -// GroupFlags combines the given flag slices together and returns the merged one. -func GroupFlags(groups ...[]cli.Flag) []cli.Flag { - var ret []cli.Flag - for _, group := range groups { - ret = append(ret, group...) - } - return ret -} - // MakeDataDir retrieves the currently requested data directory, terminating // if none (or the empty string) is specified. If the node is starting a testnet, // then a subdirectory of the specified datadir will be used. func MakeDataDir(ctx *cli.Context) string { - if path := ctx.GlobalString(DataDirFlag.Name); path != "" { - if ctx.GlobalBool(RopstenFlag.Name) { + if path := ctx.String(DataDirFlag.Name); path != "" { + if ctx.Bool(RopstenFlag.Name) { // Maintain compatibility with older Geth configurations storing the // Ropsten database in `testnet` instead of `ropsten`. return filepath.Join(path, "ropsten") } - if ctx.GlobalBool(RinkebyFlag.Name) { + if ctx.Bool(RinkebyFlag.Name) { return filepath.Join(path, "rinkeby") } - if ctx.GlobalBool(GoerliFlag.Name) { + if ctx.Bool(GoerliFlag.Name) { return filepath.Join(path, "goerli") } - if ctx.GlobalBool(SepoliaFlag.Name) { + if ctx.Bool(SepoliaFlag.Name) { return filepath.Join(path, "sepolia") } - if ctx.GlobalBool(KilnFlag.Name) { + if ctx.Bool(KilnFlag.Name) { return filepath.Join(path, "kiln") } return path @@ -895,8 +1042,8 @@ func MakeDataDir(ctx *cli.Context) string { // method returns nil and an emphemeral key is to be generated. func setNodeKey(ctx *cli.Context, cfg *p2p.Config) { var ( - hex = ctx.GlobalString(NodeKeyHexFlag.Name) - file = ctx.GlobalString(NodeKeyFileFlag.Name) + hex = ctx.String(NodeKeyHexFlag.Name) + file = ctx.String(NodeKeyFileFlag.Name) key *ecdsa.PrivateKey err error ) @@ -918,7 +1065,7 @@ func setNodeKey(ctx *cli.Context, cfg *p2p.Config) { // setNodeUserIdent creates the user identifier from CLI flags. func setNodeUserIdent(ctx *cli.Context, cfg *node.Config) { - if identity := ctx.GlobalString(IdentityFlag.Name); len(identity) > 0 { + if identity := ctx.String(IdentityFlag.Name); len(identity) > 0 { cfg.UserIdent = identity } } @@ -928,20 +1075,23 @@ func setNodeUserIdent(ctx *cli.Context, cfg *node.Config) { func setBootstrapNodes(ctx *cli.Context, cfg *p2p.Config) { urls := params.MainnetBootnodes switch { - case ctx.GlobalIsSet(BootnodesFlag.Name): - urls = SplitAndTrim(ctx.GlobalString(BootnodesFlag.Name)) - case ctx.GlobalBool(RopstenFlag.Name): + case ctx.IsSet(BootnodesFlag.Name): + urls = SplitAndTrim(ctx.String(BootnodesFlag.Name)) + case ctx.Bool(RopstenFlag.Name): urls = params.RopstenBootnodes - case ctx.GlobalBool(SepoliaFlag.Name): + case ctx.Bool(SepoliaFlag.Name): urls = params.SepoliaBootnodes - case ctx.GlobalBool(RinkebyFlag.Name): + case ctx.Bool(RinkebyFlag.Name): urls = params.RinkebyBootnodes - case ctx.GlobalBool(GoerliFlag.Name): + case ctx.Bool(GoerliFlag.Name): urls = params.GoerliBootnodes - case ctx.GlobalBool(KilnFlag.Name): + case ctx.Bool(KilnFlag.Name): urls = params.KilnBootnodes - case cfg.BootstrapNodes != nil: - return // already set, don't apply defaults. + } + + // don't apply defaults if BootstrapNodes is already set + if cfg.BootstrapNodes != nil { + return } cfg.BootstrapNodes = make([]*enode.Node, 0, len(urls)) @@ -962,8 +1112,8 @@ func setBootstrapNodes(ctx *cli.Context, cfg *p2p.Config) { func setBootstrapNodesV5(ctx *cli.Context, cfg *p2p.Config) { urls := params.V5Bootnodes switch { - case ctx.GlobalIsSet(BootnodesFlag.Name): - urls = SplitAndTrim(ctx.GlobalString(BootnodesFlag.Name)) + case ctx.IsSet(BootnodesFlag.Name): + urls = SplitAndTrim(ctx.String(BootnodesFlag.Name)) case cfg.BootstrapNodesV5 != nil: return // already set, don't apply defaults. } @@ -981,18 +1131,21 @@ func setBootstrapNodesV5(ctx *cli.Context, cfg *p2p.Config) { } } -// setListenAddress creates a TCP listening address string from set command -// line flags. +// setListenAddress creates TCP/UDP listening address strings from set command +// line flags func setListenAddress(ctx *cli.Context, cfg *p2p.Config) { - if ctx.GlobalIsSet(ListenPortFlag.Name) { - cfg.ListenAddr = fmt.Sprintf(":%d", ctx.GlobalInt(ListenPortFlag.Name)) + if ctx.IsSet(ListenPortFlag.Name) { + cfg.ListenAddr = fmt.Sprintf(":%d", ctx.Int(ListenPortFlag.Name)) + } + if ctx.IsSet(DiscoveryPortFlag.Name) { + cfg.DiscAddr = fmt.Sprintf(":%d", ctx.Int(DiscoveryPortFlag.Name)) } } // setNAT creates a port mapper from command line flags. func setNAT(ctx *cli.Context, cfg *p2p.Config) { - if ctx.GlobalIsSet(NATFlag.Name) { - natif, err := nat.Parse(ctx.GlobalString(NATFlag.Name)) + if ctx.IsSet(NATFlag.Name) { + natif, err := nat.Parse(ctx.String(NATFlag.Name)) if err != nil { Fatalf("Option %s: %v", NATFlag.Name, err) } @@ -1015,83 +1168,83 @@ func SplitAndTrim(input string) (ret []string) { // setHTTP creates the HTTP RPC listener interface string from the set // command line flags, returning empty if the HTTP endpoint is disabled. func setHTTP(ctx *cli.Context, cfg *node.Config) { - if ctx.GlobalBool(HTTPEnabledFlag.Name) && cfg.HTTPHost == "" { + if ctx.Bool(HTTPEnabledFlag.Name) && cfg.HTTPHost == "" { cfg.HTTPHost = "127.0.0.1" - if ctx.GlobalIsSet(HTTPListenAddrFlag.Name) { - cfg.HTTPHost = ctx.GlobalString(HTTPListenAddrFlag.Name) + if ctx.IsSet(HTTPListenAddrFlag.Name) { + cfg.HTTPHost = ctx.String(HTTPListenAddrFlag.Name) } } - if ctx.GlobalIsSet(HTTPPortFlag.Name) { - cfg.HTTPPort = ctx.GlobalInt(HTTPPortFlag.Name) + if ctx.IsSet(HTTPPortFlag.Name) { + cfg.HTTPPort = ctx.Int(HTTPPortFlag.Name) } - if ctx.GlobalIsSet(AuthListenFlag.Name) { - cfg.AuthAddr = ctx.GlobalString(AuthListenFlag.Name) + if ctx.IsSet(AuthListenFlag.Name) { + cfg.AuthAddr = ctx.String(AuthListenFlag.Name) } - if ctx.GlobalIsSet(AuthPortFlag.Name) { - cfg.AuthPort = ctx.GlobalInt(AuthPortFlag.Name) + if ctx.IsSet(AuthPortFlag.Name) { + cfg.AuthPort = ctx.Int(AuthPortFlag.Name) } - if ctx.GlobalIsSet(AuthVirtualHostsFlag.Name) { - cfg.AuthVirtualHosts = SplitAndTrim(ctx.GlobalString(AuthVirtualHostsFlag.Name)) + if ctx.IsSet(AuthVirtualHostsFlag.Name) { + cfg.AuthVirtualHosts = SplitAndTrim(ctx.String(AuthVirtualHostsFlag.Name)) } - if ctx.GlobalIsSet(HTTPCORSDomainFlag.Name) { - cfg.HTTPCors = SplitAndTrim(ctx.GlobalString(HTTPCORSDomainFlag.Name)) + if ctx.IsSet(HTTPCORSDomainFlag.Name) { + cfg.HTTPCors = SplitAndTrim(ctx.String(HTTPCORSDomainFlag.Name)) } - if ctx.GlobalIsSet(HTTPApiFlag.Name) { - cfg.HTTPModules = SplitAndTrim(ctx.GlobalString(HTTPApiFlag.Name)) + if ctx.IsSet(HTTPApiFlag.Name) { + cfg.HTTPModules = SplitAndTrim(ctx.String(HTTPApiFlag.Name)) } - if ctx.GlobalIsSet(HTTPVirtualHostsFlag.Name) { - cfg.HTTPVirtualHosts = SplitAndTrim(ctx.GlobalString(HTTPVirtualHostsFlag.Name)) + if ctx.IsSet(HTTPVirtualHostsFlag.Name) { + cfg.HTTPVirtualHosts = SplitAndTrim(ctx.String(HTTPVirtualHostsFlag.Name)) } - if ctx.GlobalIsSet(HTTPPathPrefixFlag.Name) { - cfg.HTTPPathPrefix = ctx.GlobalString(HTTPPathPrefixFlag.Name) + if ctx.IsSet(HTTPPathPrefixFlag.Name) { + cfg.HTTPPathPrefix = ctx.String(HTTPPathPrefixFlag.Name) } - if ctx.GlobalIsSet(AllowUnprotectedTxs.Name) { - cfg.AllowUnprotectedTxs = ctx.GlobalBool(AllowUnprotectedTxs.Name) + if ctx.IsSet(AllowUnprotectedTxs.Name) { + cfg.AllowUnprotectedTxs = ctx.Bool(AllowUnprotectedTxs.Name) } } // setGraphQL creates the GraphQL listener interface string from the set // command line flags, returning empty if the GraphQL endpoint is disabled. func setGraphQL(ctx *cli.Context, cfg *node.Config) { - if ctx.GlobalIsSet(GraphQLCORSDomainFlag.Name) { - cfg.GraphQLCors = SplitAndTrim(ctx.GlobalString(GraphQLCORSDomainFlag.Name)) + if ctx.IsSet(GraphQLCORSDomainFlag.Name) { + cfg.GraphQLCors = SplitAndTrim(ctx.String(GraphQLCORSDomainFlag.Name)) } - if ctx.GlobalIsSet(GraphQLVirtualHostsFlag.Name) { - cfg.GraphQLVirtualHosts = SplitAndTrim(ctx.GlobalString(GraphQLVirtualHostsFlag.Name)) + if ctx.IsSet(GraphQLVirtualHostsFlag.Name) { + cfg.GraphQLVirtualHosts = SplitAndTrim(ctx.String(GraphQLVirtualHostsFlag.Name)) } } // setWS creates the WebSocket RPC listener interface string from the set // command line flags, returning empty if the HTTP endpoint is disabled. func setWS(ctx *cli.Context, cfg *node.Config) { - if ctx.GlobalBool(WSEnabledFlag.Name) && cfg.WSHost == "" { + if ctx.Bool(WSEnabledFlag.Name) && cfg.WSHost == "" { cfg.WSHost = "127.0.0.1" - if ctx.GlobalIsSet(WSListenAddrFlag.Name) { - cfg.WSHost = ctx.GlobalString(WSListenAddrFlag.Name) + if ctx.IsSet(WSListenAddrFlag.Name) { + cfg.WSHost = ctx.String(WSListenAddrFlag.Name) } } - if ctx.GlobalIsSet(WSPortFlag.Name) { - cfg.WSPort = ctx.GlobalInt(WSPortFlag.Name) + if ctx.IsSet(WSPortFlag.Name) { + cfg.WSPort = ctx.Int(WSPortFlag.Name) } - if ctx.GlobalIsSet(WSAllowedOriginsFlag.Name) { - cfg.WSOrigins = SplitAndTrim(ctx.GlobalString(WSAllowedOriginsFlag.Name)) + if ctx.IsSet(WSAllowedOriginsFlag.Name) { + cfg.WSOrigins = SplitAndTrim(ctx.String(WSAllowedOriginsFlag.Name)) } - if ctx.GlobalIsSet(WSApiFlag.Name) { - cfg.WSModules = SplitAndTrim(ctx.GlobalString(WSApiFlag.Name)) + if ctx.IsSet(WSApiFlag.Name) { + cfg.WSModules = SplitAndTrim(ctx.String(WSApiFlag.Name)) } - if ctx.GlobalIsSet(WSPathPrefixFlag.Name) { - cfg.WSPathPrefix = ctx.GlobalString(WSPathPrefixFlag.Name) + if ctx.IsSet(WSPathPrefixFlag.Name) { + cfg.WSPathPrefix = ctx.String(WSPathPrefixFlag.Name) } } @@ -1100,45 +1253,45 @@ func setWS(ctx *cli.Context, cfg *node.Config) { func setIPC(ctx *cli.Context, cfg *node.Config) { CheckExclusive(ctx, IPCDisabledFlag, IPCPathFlag) switch { - case ctx.GlobalBool(IPCDisabledFlag.Name): + case ctx.Bool(IPCDisabledFlag.Name): cfg.IPCPath = "" - case ctx.GlobalIsSet(IPCPathFlag.Name): - cfg.IPCPath = ctx.GlobalString(IPCPathFlag.Name) + case ctx.IsSet(IPCPathFlag.Name): + cfg.IPCPath = ctx.String(IPCPathFlag.Name) } } // setLes configures the les server and ultra light client settings from the command line flags. func setLes(ctx *cli.Context, cfg *ethconfig.Config) { - if ctx.GlobalIsSet(LightServeFlag.Name) { - cfg.LightServ = ctx.GlobalInt(LightServeFlag.Name) + if ctx.IsSet(LightServeFlag.Name) { + cfg.LightServ = ctx.Int(LightServeFlag.Name) } - if ctx.GlobalIsSet(LightIngressFlag.Name) { - cfg.LightIngress = ctx.GlobalInt(LightIngressFlag.Name) + if ctx.IsSet(LightIngressFlag.Name) { + cfg.LightIngress = ctx.Int(LightIngressFlag.Name) } - if ctx.GlobalIsSet(LightEgressFlag.Name) { - cfg.LightEgress = ctx.GlobalInt(LightEgressFlag.Name) + if ctx.IsSet(LightEgressFlag.Name) { + cfg.LightEgress = ctx.Int(LightEgressFlag.Name) } - if ctx.GlobalIsSet(LightMaxPeersFlag.Name) { - cfg.LightPeers = ctx.GlobalInt(LightMaxPeersFlag.Name) + if ctx.IsSet(LightMaxPeersFlag.Name) { + cfg.LightPeers = ctx.Int(LightMaxPeersFlag.Name) } - if ctx.GlobalIsSet(UltraLightServersFlag.Name) { - cfg.UltraLightServers = strings.Split(ctx.GlobalString(UltraLightServersFlag.Name), ",") + if ctx.IsSet(UltraLightServersFlag.Name) { + cfg.UltraLightServers = strings.Split(ctx.String(UltraLightServersFlag.Name), ",") } - if ctx.GlobalIsSet(UltraLightFractionFlag.Name) { - cfg.UltraLightFraction = ctx.GlobalInt(UltraLightFractionFlag.Name) + if ctx.IsSet(UltraLightFractionFlag.Name) { + cfg.UltraLightFraction = ctx.Int(UltraLightFractionFlag.Name) } if cfg.UltraLightFraction <= 0 && cfg.UltraLightFraction > 100 { log.Error("Ultra light fraction is invalid", "had", cfg.UltraLightFraction, "updated", ethconfig.Defaults.UltraLightFraction) cfg.UltraLightFraction = ethconfig.Defaults.UltraLightFraction } - if ctx.GlobalIsSet(UltraLightOnlyAnnounceFlag.Name) { - cfg.UltraLightOnlyAnnounce = ctx.GlobalBool(UltraLightOnlyAnnounceFlag.Name) + if ctx.IsSet(UltraLightOnlyAnnounceFlag.Name) { + cfg.UltraLightOnlyAnnounce = ctx.Bool(UltraLightOnlyAnnounceFlag.Name) } - if ctx.GlobalIsSet(LightNoPruneFlag.Name) { - cfg.LightNoPrune = ctx.GlobalBool(LightNoPruneFlag.Name) + if ctx.IsSet(LightNoPruneFlag.Name) { + cfg.LightNoPrune = ctx.Bool(LightNoPruneFlag.Name) } - if ctx.GlobalIsSet(LightNoSyncServeFlag.Name) { - cfg.LightNoSyncServe = ctx.GlobalBool(LightNoSyncServeFlag.Name) + if ctx.IsSet(LightNoSyncServeFlag.Name) { + cfg.LightNoSyncServe = ctx.Bool(LightNoSyncServeFlag.Name) } } @@ -1199,8 +1352,8 @@ func MakeAddress(ks *keystore.KeyStore, account string) (accounts.Account, error func setEtherbase(ctx *cli.Context, ks *keystore.KeyStore, cfg *ethconfig.Config) { // Extract the current etherbase var etherbase string - if ctx.GlobalIsSet(MinerEtherbaseFlag.Name) { - etherbase = ctx.GlobalString(MinerEtherbaseFlag.Name) + if ctx.IsSet(MinerEtherbaseFlag.Name) { + etherbase = ctx.String(MinerEtherbaseFlag.Name) } // Convert the etherbase into an address and configure it if etherbase != "" { @@ -1218,7 +1371,7 @@ func setEtherbase(ctx *cli.Context, ks *keystore.KeyStore, cfg *ethconfig.Config // MakePasswordList reads password lines from the file specified by the global --password flag. func MakePasswordList(ctx *cli.Context) []string { - path := ctx.GlobalString(PasswordFileFlag.Name) + path := ctx.Path(PasswordFileFlag.Name) if path == "" { return nil } @@ -1241,25 +1394,25 @@ func SetP2PConfig(ctx *cli.Context, cfg *p2p.Config) { setBootstrapNodes(ctx, cfg) setBootstrapNodesV5(ctx, cfg) - lightClient := ctx.GlobalString(SyncModeFlag.Name) == "light" - lightServer := (ctx.GlobalInt(LightServeFlag.Name) != 0) + lightClient := ctx.String(SyncModeFlag.Name) == "light" + lightServer := (ctx.Int(LightServeFlag.Name) != 0) - lightPeers := ctx.GlobalInt(LightMaxPeersFlag.Name) - if lightClient && !ctx.GlobalIsSet(LightMaxPeersFlag.Name) { + lightPeers := ctx.Int(LightMaxPeersFlag.Name) + if lightClient && !ctx.IsSet(LightMaxPeersFlag.Name) { // dynamic default - for clients we use 1/10th of the default for servers lightPeers /= 10 } - if ctx.GlobalIsSet(MaxPeersFlag.Name) { - cfg.MaxPeers = ctx.GlobalInt(MaxPeersFlag.Name) - if lightServer && !ctx.GlobalIsSet(LightMaxPeersFlag.Name) { + if ctx.IsSet(MaxPeersFlag.Name) { + cfg.MaxPeers = ctx.Int(MaxPeersFlag.Name) + if lightServer && !ctx.IsSet(LightMaxPeersFlag.Name) { cfg.MaxPeers += lightPeers } } else { if lightServer { cfg.MaxPeers += lightPeers } - if lightClient && ctx.GlobalIsSet(LightMaxPeersFlag.Name) && cfg.MaxPeers < lightPeers { + if lightClient && ctx.IsSet(LightMaxPeersFlag.Name) && cfg.MaxPeers < lightPeers { cfg.MaxPeers = lightPeers } } @@ -1272,24 +1425,24 @@ func SetP2PConfig(ctx *cli.Context, cfg *p2p.Config) { } log.Info("Maximum peer count", "ETH", ethPeers, "LES", lightPeers, "total", cfg.MaxPeers) - if ctx.GlobalIsSet(MaxPendingPeersFlag.Name) { - cfg.MaxPendingPeers = ctx.GlobalInt(MaxPendingPeersFlag.Name) + if ctx.IsSet(MaxPendingPeersFlag.Name) { + cfg.MaxPendingPeers = ctx.Int(MaxPendingPeersFlag.Name) } - if ctx.GlobalIsSet(NoDiscoverFlag.Name) || lightClient { + if ctx.IsSet(NoDiscoverFlag.Name) || lightClient { cfg.NoDiscovery = true } // if we're running a light client or server, force enable the v5 peer discovery // unless it is explicitly disabled with --nodiscover note that explicitly specifying // --v5disc overrides --nodiscover, in which case the later only disables v4 discovery - forceV5Discovery := (lightClient || lightServer) && !ctx.GlobalBool(NoDiscoverFlag.Name) - if ctx.GlobalIsSet(DiscoveryV5Flag.Name) { - cfg.DiscoveryV5 = ctx.GlobalBool(DiscoveryV5Flag.Name) + forceV5Discovery := (lightClient || lightServer) && !ctx.Bool(NoDiscoverFlag.Name) + if ctx.IsSet(DiscoveryV5Flag.Name) { + cfg.DiscoveryV5 = ctx.Bool(DiscoveryV5Flag.Name) } else if forceV5Discovery { cfg.DiscoveryV5 = true } - if netrestrict := ctx.GlobalString(NetrestrictFlag.Name); netrestrict != "" { + if netrestrict := ctx.String(NetrestrictFlag.Name); netrestrict != "" { list, err := netutil.ParseNetlist(netrestrict) if err != nil { Fatalf("Option %q: %v", NetrestrictFlag.Name, err) @@ -1297,7 +1450,7 @@ func SetP2PConfig(ctx *cli.Context, cfg *p2p.Config) { cfg.NetRestrict = list } - if ctx.GlobalBool(DeveloperFlag.Name) { + if ctx.Bool(DeveloperFlag.Name) { // --dev mode can't use p2p networking. cfg.MaxPeers = 0 cfg.ListenAddr = "" @@ -1315,40 +1468,40 @@ func SetNodeConfig(ctx *cli.Context, cfg *node.Config) { setGraphQL(ctx, cfg) setWS(ctx, cfg) setNodeUserIdent(ctx, cfg) - setDataDir(ctx, cfg) + SetDataDir(ctx, cfg) setSmartCard(ctx, cfg) - if ctx.GlobalIsSet(JWTSecretFlag.Name) { - cfg.JWTSecret = ctx.GlobalString(JWTSecretFlag.Name) + if ctx.IsSet(JWTSecretFlag.Name) { + cfg.JWTSecret = ctx.String(JWTSecretFlag.Name) } - if ctx.GlobalIsSet(ExternalSignerFlag.Name) { - cfg.ExternalSigner = ctx.GlobalString(ExternalSignerFlag.Name) + if ctx.IsSet(ExternalSignerFlag.Name) { + cfg.ExternalSigner = ctx.String(ExternalSignerFlag.Name) } - if ctx.GlobalIsSet(KeyStoreDirFlag.Name) { - cfg.KeyStoreDir = ctx.GlobalString(KeyStoreDirFlag.Name) + if ctx.IsSet(KeyStoreDirFlag.Name) { + cfg.KeyStoreDir = ctx.String(KeyStoreDirFlag.Name) } - if ctx.GlobalIsSet(DeveloperFlag.Name) { + if ctx.IsSet(DeveloperFlag.Name) { cfg.UseLightweightKDF = true } - if ctx.GlobalIsSet(LightKDFFlag.Name) { - cfg.UseLightweightKDF = ctx.GlobalBool(LightKDFFlag.Name) + if ctx.IsSet(LightKDFFlag.Name) { + cfg.UseLightweightKDF = ctx.Bool(LightKDFFlag.Name) } - if ctx.GlobalIsSet(NoUSBFlag.Name) || cfg.NoUSB { + if ctx.IsSet(NoUSBFlag.Name) || cfg.NoUSB { log.Warn("Option nousb is deprecated and USB is deactivated by default. Use --usb to enable") } - if ctx.GlobalIsSet(USBFlag.Name) { - cfg.USB = ctx.GlobalBool(USBFlag.Name) + if ctx.IsSet(USBFlag.Name) { + cfg.USB = ctx.Bool(USBFlag.Name) } - if ctx.GlobalIsSet(InsecureUnlockAllowedFlag.Name) { - cfg.InsecureUnlockAllowed = ctx.GlobalBool(InsecureUnlockAllowedFlag.Name) + if ctx.IsSet(InsecureUnlockAllowedFlag.Name) { + cfg.InsecureUnlockAllowed = ctx.Bool(InsecureUnlockAllowedFlag.Name) } } func setSmartCard(ctx *cli.Context, cfg *node.Config) { // Skip enabling smartcards if no path is set - path := ctx.GlobalString(SmartCardDaemonPathFlag.Name) + path := ctx.String(SmartCardDaemonPathFlag.Name) if path == "" { return } @@ -1366,13 +1519,13 @@ func setSmartCard(ctx *cli.Context, cfg *node.Config) { cfg.SmartCardDaemonPath = path } -func setDataDir(ctx *cli.Context, cfg *node.Config) { +func SetDataDir(ctx *cli.Context, cfg *node.Config) { switch { - case ctx.GlobalIsSet(DataDirFlag.Name): - cfg.DataDir = ctx.GlobalString(DataDirFlag.Name) - case ctx.GlobalBool(DeveloperFlag.Name): + case ctx.IsSet(DataDirFlag.Name): + cfg.DataDir = ctx.String(DataDirFlag.Name) + case ctx.Bool(DeveloperFlag.Name): cfg.DataDir = "" // unless explicitly requested, use memory databases - case ctx.GlobalBool(RopstenFlag.Name) && cfg.DataDir == node.DefaultDataDir(): + case ctx.Bool(RopstenFlag.Name) && cfg.DataDir == node.DefaultDataDir(): // Maintain compatibility with older Geth configurations storing the // Ropsten database in `testnet` instead of `ropsten`. legacyPath := filepath.Join(node.DefaultDataDir(), "testnet") @@ -1384,13 +1537,13 @@ func setDataDir(ctx *cli.Context, cfg *node.Config) { } cfg.DataDir = filepath.Join(node.DefaultDataDir(), "ropsten") - case ctx.GlobalBool(RinkebyFlag.Name) && cfg.DataDir == node.DefaultDataDir(): + case ctx.Bool(RinkebyFlag.Name) && cfg.DataDir == node.DefaultDataDir(): cfg.DataDir = filepath.Join(node.DefaultDataDir(), "rinkeby") - case ctx.GlobalBool(GoerliFlag.Name) && cfg.DataDir == node.DefaultDataDir(): + case ctx.Bool(GoerliFlag.Name) && cfg.DataDir == node.DefaultDataDir(): cfg.DataDir = filepath.Join(node.DefaultDataDir(), "goerli") - case ctx.GlobalBool(SepoliaFlag.Name) && cfg.DataDir == node.DefaultDataDir(): + case ctx.Bool(SepoliaFlag.Name) && cfg.DataDir == node.DefaultDataDir(): cfg.DataDir = filepath.Join(node.DefaultDataDir(), "sepolia") - case ctx.GlobalBool(KilnFlag.Name) && cfg.DataDir == node.DefaultDataDir(): + case ctx.Bool(KilnFlag.Name) && cfg.DataDir == node.DefaultDataDir(): cfg.DataDir = filepath.Join(node.DefaultDataDir(), "kiln") } } @@ -1401,23 +1554,23 @@ func setGPO(ctx *cli.Context, cfg *gasprice.Config, light bool) { if light { *cfg = ethconfig.LightClientGPO } - if ctx.GlobalIsSet(GpoBlocksFlag.Name) { - cfg.Blocks = ctx.GlobalInt(GpoBlocksFlag.Name) + if ctx.IsSet(GpoBlocksFlag.Name) { + cfg.Blocks = ctx.Int(GpoBlocksFlag.Name) } - if ctx.GlobalIsSet(GpoPercentileFlag.Name) { - cfg.Percentile = ctx.GlobalInt(GpoPercentileFlag.Name) + if ctx.IsSet(GpoPercentileFlag.Name) { + cfg.Percentile = ctx.Int(GpoPercentileFlag.Name) } - if ctx.GlobalIsSet(GpoMaxGasPriceFlag.Name) { - cfg.MaxPrice = big.NewInt(ctx.GlobalInt64(GpoMaxGasPriceFlag.Name)) + if ctx.IsSet(GpoMaxGasPriceFlag.Name) { + cfg.MaxPrice = big.NewInt(ctx.Int64(GpoMaxGasPriceFlag.Name)) } - if ctx.GlobalIsSet(GpoIgnoreGasPriceFlag.Name) { - cfg.IgnorePrice = big.NewInt(ctx.GlobalInt64(GpoIgnoreGasPriceFlag.Name)) + if ctx.IsSet(GpoIgnoreGasPriceFlag.Name) { + cfg.IgnorePrice = big.NewInt(ctx.Int64(GpoIgnoreGasPriceFlag.Name)) } } func setTxPool(ctx *cli.Context, cfg *core.TxPoolConfig) { - if ctx.GlobalIsSet(TxPoolLocalsFlag.Name) { - locals := strings.Split(ctx.GlobalString(TxPoolLocalsFlag.Name), ",") + if ctx.IsSet(TxPoolLocalsFlag.Name) { + locals := strings.Split(ctx.String(TxPoolLocalsFlag.Name), ",") for _, account := range locals { if trimmed := strings.TrimSpace(account); !common.IsHexAddress(trimmed) { Fatalf("Invalid account in --txpool.locals: %s", trimmed) @@ -1426,96 +1579,93 @@ func setTxPool(ctx *cli.Context, cfg *core.TxPoolConfig) { } } } - if ctx.GlobalIsSet(TxPoolNoLocalsFlag.Name) { - cfg.NoLocals = ctx.GlobalBool(TxPoolNoLocalsFlag.Name) + if ctx.IsSet(TxPoolNoLocalsFlag.Name) { + cfg.NoLocals = ctx.Bool(TxPoolNoLocalsFlag.Name) } - if ctx.GlobalIsSet(TxPoolJournalFlag.Name) { - cfg.Journal = ctx.GlobalString(TxPoolJournalFlag.Name) + if ctx.IsSet(TxPoolJournalFlag.Name) { + cfg.Journal = ctx.String(TxPoolJournalFlag.Name) } - if ctx.GlobalIsSet(TxPoolRejournalFlag.Name) { - cfg.Rejournal = ctx.GlobalDuration(TxPoolRejournalFlag.Name) + if ctx.IsSet(TxPoolRejournalFlag.Name) { + cfg.Rejournal = ctx.Duration(TxPoolRejournalFlag.Name) } - if ctx.GlobalIsSet(TxPoolPriceLimitFlag.Name) { - cfg.PriceLimit = ctx.GlobalUint64(TxPoolPriceLimitFlag.Name) + if ctx.IsSet(TxPoolPriceLimitFlag.Name) { + cfg.PriceLimit = ctx.Uint64(TxPoolPriceLimitFlag.Name) } - if ctx.GlobalIsSet(TxPoolPriceBumpFlag.Name) { - cfg.PriceBump = ctx.GlobalUint64(TxPoolPriceBumpFlag.Name) + if ctx.IsSet(TxPoolPriceBumpFlag.Name) { + cfg.PriceBump = ctx.Uint64(TxPoolPriceBumpFlag.Name) } - if ctx.GlobalIsSet(TxPoolAccountSlotsFlag.Name) { - cfg.AccountSlots = ctx.GlobalUint64(TxPoolAccountSlotsFlag.Name) + if ctx.IsSet(TxPoolAccountSlotsFlag.Name) { + cfg.AccountSlots = ctx.Uint64(TxPoolAccountSlotsFlag.Name) } - if ctx.GlobalIsSet(TxPoolGlobalSlotsFlag.Name) { - cfg.GlobalSlots = ctx.GlobalUint64(TxPoolGlobalSlotsFlag.Name) + if ctx.IsSet(TxPoolGlobalSlotsFlag.Name) { + cfg.GlobalSlots = ctx.Uint64(TxPoolGlobalSlotsFlag.Name) } - if ctx.GlobalIsSet(TxPoolAccountQueueFlag.Name) { - cfg.AccountQueue = ctx.GlobalUint64(TxPoolAccountQueueFlag.Name) + if ctx.IsSet(TxPoolAccountQueueFlag.Name) { + cfg.AccountQueue = ctx.Uint64(TxPoolAccountQueueFlag.Name) } - if ctx.GlobalIsSet(TxPoolGlobalQueueFlag.Name) { - cfg.GlobalQueue = ctx.GlobalUint64(TxPoolGlobalQueueFlag.Name) + if ctx.IsSet(TxPoolGlobalQueueFlag.Name) { + cfg.GlobalQueue = ctx.Uint64(TxPoolGlobalQueueFlag.Name) } - if ctx.GlobalIsSet(TxPoolLifetimeFlag.Name) { - cfg.Lifetime = ctx.GlobalDuration(TxPoolLifetimeFlag.Name) + if ctx.IsSet(TxPoolLifetimeFlag.Name) { + cfg.Lifetime = ctx.Duration(TxPoolLifetimeFlag.Name) } } func setEthash(ctx *cli.Context, cfg *ethconfig.Config) { - if ctx.GlobalIsSet(EthashCacheDirFlag.Name) { - cfg.Ethash.CacheDir = ctx.GlobalString(EthashCacheDirFlag.Name) + if ctx.IsSet(EthashCacheDirFlag.Name) { + cfg.Ethash.CacheDir = ctx.String(EthashCacheDirFlag.Name) } - if ctx.GlobalIsSet(EthashDatasetDirFlag.Name) { - cfg.Ethash.DatasetDir = ctx.GlobalString(EthashDatasetDirFlag.Name) + if ctx.IsSet(EthashDatasetDirFlag.Name) { + cfg.Ethash.DatasetDir = ctx.String(EthashDatasetDirFlag.Name) } - if ctx.GlobalIsSet(EthashCachesInMemoryFlag.Name) { - cfg.Ethash.CachesInMem = ctx.GlobalInt(EthashCachesInMemoryFlag.Name) + if ctx.IsSet(EthashCachesInMemoryFlag.Name) { + cfg.Ethash.CachesInMem = ctx.Int(EthashCachesInMemoryFlag.Name) } - if ctx.GlobalIsSet(EthashCachesOnDiskFlag.Name) { - cfg.Ethash.CachesOnDisk = ctx.GlobalInt(EthashCachesOnDiskFlag.Name) + if ctx.IsSet(EthashCachesOnDiskFlag.Name) { + cfg.Ethash.CachesOnDisk = ctx.Int(EthashCachesOnDiskFlag.Name) } - if ctx.GlobalIsSet(EthashCachesLockMmapFlag.Name) { - cfg.Ethash.CachesLockMmap = ctx.GlobalBool(EthashCachesLockMmapFlag.Name) + if ctx.IsSet(EthashCachesLockMmapFlag.Name) { + cfg.Ethash.CachesLockMmap = ctx.Bool(EthashCachesLockMmapFlag.Name) } - if ctx.GlobalIsSet(EthashDatasetsInMemoryFlag.Name) { - cfg.Ethash.DatasetsInMem = ctx.GlobalInt(EthashDatasetsInMemoryFlag.Name) + if ctx.IsSet(EthashDatasetsInMemoryFlag.Name) { + cfg.Ethash.DatasetsInMem = ctx.Int(EthashDatasetsInMemoryFlag.Name) } - if ctx.GlobalIsSet(EthashDatasetsOnDiskFlag.Name) { - cfg.Ethash.DatasetsOnDisk = ctx.GlobalInt(EthashDatasetsOnDiskFlag.Name) + if ctx.IsSet(EthashDatasetsOnDiskFlag.Name) { + cfg.Ethash.DatasetsOnDisk = ctx.Int(EthashDatasetsOnDiskFlag.Name) } - if ctx.GlobalIsSet(EthashDatasetsLockMmapFlag.Name) { - cfg.Ethash.DatasetsLockMmap = ctx.GlobalBool(EthashDatasetsLockMmapFlag.Name) + if ctx.IsSet(EthashDatasetsLockMmapFlag.Name) { + cfg.Ethash.DatasetsLockMmap = ctx.Bool(EthashDatasetsLockMmapFlag.Name) } } func setMiner(ctx *cli.Context, cfg *miner.Config) { - if ctx.GlobalIsSet(MinerNotifyFlag.Name) { - cfg.Notify = strings.Split(ctx.GlobalString(MinerNotifyFlag.Name), ",") + if ctx.IsSet(MinerNotifyFlag.Name) { + cfg.Notify = strings.Split(ctx.String(MinerNotifyFlag.Name), ",") } - cfg.NotifyFull = ctx.GlobalBool(MinerNotifyFullFlag.Name) - if ctx.GlobalIsSet(MinerExtraDataFlag.Name) { - cfg.ExtraData = []byte(ctx.GlobalString(MinerExtraDataFlag.Name)) + cfg.NotifyFull = ctx.Bool(MinerNotifyFullFlag.Name) + if ctx.IsSet(MinerExtraDataFlag.Name) { + cfg.ExtraData = []byte(ctx.String(MinerExtraDataFlag.Name)) } - if ctx.GlobalIsSet(MinerGasLimitFlag.Name) { - cfg.GasCeil = ctx.GlobalUint64(MinerGasLimitFlag.Name) + if ctx.IsSet(MinerGasLimitFlag.Name) { + cfg.GasCeil = ctx.Uint64(MinerGasLimitFlag.Name) } - if ctx.GlobalIsSet(MinerGasPriceFlag.Name) { - cfg.GasPrice = GlobalBig(ctx, MinerGasPriceFlag.Name) + if ctx.IsSet(MinerGasPriceFlag.Name) { + cfg.GasPrice = flags.GlobalBig(ctx, MinerGasPriceFlag.Name) } - if ctx.GlobalIsSet(MinerRecommitIntervalFlag.Name) { - cfg.Recommit = ctx.GlobalDuration(MinerRecommitIntervalFlag.Name) + if ctx.IsSet(MinerRecommitIntervalFlag.Name) { + cfg.Recommit = ctx.Duration(MinerRecommitIntervalFlag.Name) } - if ctx.GlobalIsSet(MinerNoVerifyFlag.Name) { - cfg.Noverify = ctx.GlobalBool(MinerNoVerifyFlag.Name) - } - if ctx.GlobalIsSet(LegacyMinerGasTargetFlag.Name) { - log.Warn("The generic --miner.gastarget flag is deprecated and will be removed in the future!") + if ctx.IsSet(MinerNoVerifyFlag.Name) { + cfg.Noverify = ctx.Bool(MinerNoVerifyFlag.Name) } } func setRequiredBlocks(ctx *cli.Context, cfg *ethconfig.Config) { - requiredBlocks := ctx.GlobalString(EthRequiredBlocksFlag.Name) + requiredBlocks := ctx.String(EthRequiredBlocksFlag.Name) if requiredBlocks == "" { - if ctx.GlobalIsSet(LegacyWhitelistFlag.Name) { + if ctx.IsSet(LegacyWhitelistFlag.Name) { log.Warn("The flag --whitelist is deprecated and will be removed, please use --eth.requiredblocks") - requiredBlocks = ctx.GlobalString(LegacyWhitelistFlag.Name) + requiredBlocks = ctx.String(LegacyWhitelistFlag.Name) } else { return } @@ -1550,13 +1700,13 @@ func CheckExclusive(ctx *cli.Context, args ...interface{}) { panic(fmt.Sprintf("invalid argument, not cli.Flag type: %T", args[i])) } // Check if next arg extends current and expand its name if so - name := flag.GetName() + name := flag.Names()[0] if i+1 < len(args) { switch option := args[i+1].(type) { case string: // Extended flag check, make sure value set doesn't conflict with passed in option - if ctx.GlobalString(flag.GetName()) == option { + if ctx.String(flag.Names()[0]) == option { name += "=" + option set = append(set, "--"+name) } @@ -1570,7 +1720,7 @@ func CheckExclusive(ctx *cli.Context, args ...interface{}) { } } // Mark the flag if it's set - if ctx.GlobalIsSet(flag.GetName()) { + if ctx.IsSet(flag.Names()[0]) { set = append(set, "--"+name) } } @@ -1585,11 +1735,11 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { CheckExclusive(ctx, MainnetFlag, DeveloperFlag, RopstenFlag, RinkebyFlag, GoerliFlag, SepoliaFlag, KilnFlag) CheckExclusive(ctx, LightServeFlag, SyncModeFlag, "light") CheckExclusive(ctx, DeveloperFlag, ExternalSignerFlag) // Can't use both ephemeral unlocked and external signer - if ctx.GlobalString(GCModeFlag.Name) == "archive" && ctx.GlobalUint64(TxLookupLimitFlag.Name) != 0 { - ctx.GlobalSet(TxLookupLimitFlag.Name, "0") + if ctx.String(GCModeFlag.Name) == "archive" && ctx.Uint64(TxLookupLimitFlag.Name) != 0 { + ctx.Set(TxLookupLimitFlag.Name, "0") log.Warn("Disable transaction unindexing for archive node") } - if ctx.GlobalIsSet(LightServeFlag.Name) && ctx.GlobalUint64(TxLookupLimitFlag.Name) != 0 { + if ctx.IsSet(LightServeFlag.Name) && ctx.Uint64(TxLookupLimitFlag.Name) != 0 { log.Warn("LES server cannot serve old transaction status and cannot connect below les/4 protocol version if transaction lookup index is limited") } var ks *keystore.KeyStore @@ -1597,7 +1747,7 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { ks = keystores[0].(*keystore.KeyStore) } setEtherbase(ctx, ks, cfg) - setGPO(ctx, &cfg.GPO, ctx.GlobalString(SyncModeFlag.Name) == "light") + setGPO(ctx, &cfg.GPO, ctx.String(SyncModeFlag.Name) == "light") setTxPool(ctx, &cfg.TxPool) setEthash(ctx, cfg) setMiner(ctx, &cfg.Miner) @@ -1612,66 +1762,69 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { mem.Total = 2 * 1024 * 1024 * 1024 } allowance := int(mem.Total / 1024 / 1024 / 3) - if cache := ctx.GlobalInt(CacheFlag.Name); cache > allowance { + if cache := ctx.Int(CacheFlag.Name); cache > allowance { log.Warn("Sanitizing cache to Go's GC limits", "provided", cache, "updated", allowance) - ctx.GlobalSet(CacheFlag.Name, strconv.Itoa(allowance)) + ctx.Set(CacheFlag.Name, strconv.Itoa(allowance)) } } // Ensure Go's GC ignores the database cache for trigger percentage - cache := ctx.GlobalInt(CacheFlag.Name) + cache := ctx.Int(CacheFlag.Name) gogc := math.Max(20, math.Min(100, 100/(float64(cache)/1024))) log.Debug("Sanitizing Go's GC trigger", "percent", int(gogc)) godebug.SetGCPercent(int(gogc)) - if ctx.GlobalIsSet(SyncModeFlag.Name) { - cfg.SyncMode = *GlobalTextMarshaler(ctx, SyncModeFlag.Name).(*downloader.SyncMode) + if ctx.IsSet(SyncModeFlag.Name) { + cfg.SyncMode = *flags.GlobalTextMarshaler(ctx, SyncModeFlag.Name).(*downloader.SyncMode) } - if ctx.GlobalIsSet(NetworkIdFlag.Name) { - cfg.NetworkId = ctx.GlobalUint64(NetworkIdFlag.Name) + if ctx.IsSet(NetworkIdFlag.Name) { + cfg.NetworkId = ctx.Uint64(NetworkIdFlag.Name) } - if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheDatabaseFlag.Name) { - cfg.DatabaseCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheDatabaseFlag.Name) / 100 + if ctx.IsSet(CacheFlag.Name) || ctx.IsSet(CacheDatabaseFlag.Name) { + cfg.DatabaseCache = ctx.Int(CacheFlag.Name) * ctx.Int(CacheDatabaseFlag.Name) / 100 } - cfg.DatabaseHandles = MakeDatabaseHandles(ctx.GlobalInt(FDLimitFlag.Name)) - if ctx.GlobalIsSet(AncientFlag.Name) { - cfg.DatabaseFreezer = ctx.GlobalString(AncientFlag.Name) + cfg.DatabaseHandles = MakeDatabaseHandles(ctx.Int(FDLimitFlag.Name)) + if ctx.IsSet(AncientFlag.Name) { + cfg.DatabaseFreezer = ctx.String(AncientFlag.Name) } - if gcmode := ctx.GlobalString(GCModeFlag.Name); gcmode != "full" && gcmode != "archive" { + if gcmode := ctx.String(GCModeFlag.Name); gcmode != "full" && gcmode != "archive" { Fatalf("--%s must be either 'full' or 'archive'", GCModeFlag.Name) } - if ctx.GlobalIsSet(GCModeFlag.Name) { - cfg.NoPruning = ctx.GlobalString(GCModeFlag.Name) == "archive" + if ctx.IsSet(GCModeFlag.Name) { + cfg.NoPruning = ctx.String(GCModeFlag.Name) == "archive" } - if ctx.GlobalIsSet(CacheNoPrefetchFlag.Name) { - cfg.NoPrefetch = ctx.GlobalBool(CacheNoPrefetchFlag.Name) + if ctx.IsSet(CacheNoPrefetchFlag.Name) { + cfg.NoPrefetch = ctx.Bool(CacheNoPrefetchFlag.Name) } // Read the value from the flag no matter if it's set or not. - cfg.Preimages = ctx.GlobalBool(CachePreimagesFlag.Name) + cfg.Preimages = ctx.Bool(CachePreimagesFlag.Name) if cfg.NoPruning && !cfg.Preimages { cfg.Preimages = true log.Info("Enabling recording of key preimages since archive mode is used") } - if ctx.GlobalIsSet(TxLookupLimitFlag.Name) { - cfg.TxLookupLimit = ctx.GlobalUint64(TxLookupLimitFlag.Name) + if ctx.IsSet(TxLookupLimitFlag.Name) { + cfg.TxLookupLimit = ctx.Uint64(TxLookupLimitFlag.Name) + } + if ctx.IsSet(CacheFlag.Name) || ctx.IsSet(CacheTrieFlag.Name) { + cfg.TrieCleanCache = ctx.Int(CacheFlag.Name) * ctx.Int(CacheTrieFlag.Name) / 100 } - if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheTrieFlag.Name) { - cfg.TrieCleanCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheTrieFlag.Name) / 100 + if ctx.IsSet(CacheTrieJournalFlag.Name) { + cfg.TrieCleanCacheJournal = ctx.String(CacheTrieJournalFlag.Name) } - if ctx.GlobalIsSet(CacheTrieJournalFlag.Name) { - cfg.TrieCleanCacheJournal = ctx.GlobalString(CacheTrieJournalFlag.Name) + if ctx.IsSet(CacheTrieRejournalFlag.Name) { + cfg.TrieCleanCacheRejournal = ctx.Duration(CacheTrieRejournalFlag.Name) } - if ctx.GlobalIsSet(CacheTrieRejournalFlag.Name) { - cfg.TrieCleanCacheRejournal = ctx.GlobalDuration(CacheTrieRejournalFlag.Name) + if ctx.IsSet(CacheFlag.Name) || ctx.IsSet(CacheGCFlag.Name) { + cfg.TrieDirtyCache = ctx.Int(CacheFlag.Name) * ctx.Int(CacheGCFlag.Name) / 100 } - if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheGCFlag.Name) { - cfg.TrieDirtyCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100 + if ctx.IsSet(CacheFlag.Name) || ctx.IsSet(CacheSnapshotFlag.Name) { + cfg.SnapshotCache = ctx.Int(CacheFlag.Name) * ctx.Int(CacheSnapshotFlag.Name) / 100 } - if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheSnapshotFlag.Name) { - cfg.SnapshotCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheSnapshotFlag.Name) / 100 + if ctx.IsSet(CacheLogSizeFlag.Name) { + cfg.FilterLogCacheSize = ctx.Int(CacheLogSizeFlag.Name) } - if !ctx.GlobalBool(SnapshotFlag.Name) { + if !ctx.Bool(SnapshotFlag.Name) { // If snap-sync is requested, this flag is also required if cfg.SyncMode == downloader.SnapSync { log.Info("Snap sync requested, enabling --snapshot") @@ -1680,32 +1833,32 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { cfg.SnapshotCache = 0 // Disabled } } - if ctx.GlobalIsSet(DocRootFlag.Name) { - cfg.DocRoot = ctx.GlobalString(DocRootFlag.Name) + if ctx.IsSet(DocRootFlag.Name) { + cfg.DocRoot = ctx.String(DocRootFlag.Name) } - if ctx.GlobalIsSet(VMEnableDebugFlag.Name) { + if ctx.IsSet(VMEnableDebugFlag.Name) { // TODO(fjl): force-enable this in --dev mode - cfg.EnablePreimageRecording = ctx.GlobalBool(VMEnableDebugFlag.Name) + cfg.EnablePreimageRecording = ctx.Bool(VMEnableDebugFlag.Name) } - if ctx.GlobalIsSet(RPCGlobalGasCapFlag.Name) { - cfg.RPCGasCap = ctx.GlobalUint64(RPCGlobalGasCapFlag.Name) + if ctx.IsSet(RPCGlobalGasCapFlag.Name) { + cfg.RPCGasCap = ctx.Uint64(RPCGlobalGasCapFlag.Name) } if cfg.RPCGasCap != 0 { log.Info("Set global gas cap", "cap", cfg.RPCGasCap) } else { log.Info("Global gas cap disabled") } - if ctx.GlobalIsSet(RPCGlobalEVMTimeoutFlag.Name) { - cfg.RPCEVMTimeout = ctx.GlobalDuration(RPCGlobalEVMTimeoutFlag.Name) + if ctx.IsSet(RPCGlobalEVMTimeoutFlag.Name) { + cfg.RPCEVMTimeout = ctx.Duration(RPCGlobalEVMTimeoutFlag.Name) } - if ctx.GlobalIsSet(RPCGlobalTxFeeCapFlag.Name) { - cfg.RPCTxFeeCap = ctx.GlobalFloat64(RPCGlobalTxFeeCapFlag.Name) + if ctx.IsSet(RPCGlobalTxFeeCapFlag.Name) { + cfg.RPCTxFeeCap = ctx.Float64(RPCGlobalTxFeeCapFlag.Name) } - if ctx.GlobalIsSet(NoDiscoverFlag.Name) { + if ctx.IsSet(NoDiscoverFlag.Name) { cfg.EthDiscoveryURLs, cfg.SnapDiscoveryURLs = []string{}, []string{} - } else if ctx.GlobalIsSet(DNSDiscoveryFlag.Name) { - urls := ctx.GlobalString(DNSDiscoveryFlag.Name) + } else if ctx.IsSet(DNSDiscoveryFlag.Name) { + urls := ctx.String(DNSDiscoveryFlag.Name) if urls == "" { cfg.EthDiscoveryURLs = []string{} } else { @@ -1714,25 +1867,25 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { } // Override any default configs for hard coded networks. switch { - case ctx.GlobalBool(MainnetFlag.Name): - if !ctx.GlobalIsSet(NetworkIdFlag.Name) { + case ctx.Bool(MainnetFlag.Name): + if !ctx.IsSet(NetworkIdFlag.Name) { cfg.NetworkId = 1 } cfg.Genesis = core.DefaultGenesisBlock() SetDNSDiscoveryDefaults(cfg, params.MainnetGenesisHash) - case ctx.GlobalBool(RopstenFlag.Name): - if !ctx.GlobalIsSet(NetworkIdFlag.Name) { + case ctx.Bool(RopstenFlag.Name): + if !ctx.IsSet(NetworkIdFlag.Name) { cfg.NetworkId = 3 } cfg.Genesis = core.DefaultRopstenGenesisBlock() SetDNSDiscoveryDefaults(cfg, params.RopstenGenesisHash) - case ctx.GlobalBool(SepoliaFlag.Name): - if !ctx.GlobalIsSet(NetworkIdFlag.Name) { + case ctx.Bool(SepoliaFlag.Name): + if !ctx.IsSet(NetworkIdFlag.Name) { cfg.NetworkId = 11155111 } cfg.Genesis = core.DefaultSepoliaGenesisBlock() SetDNSDiscoveryDefaults(cfg, params.SepoliaGenesisHash) - case ctx.GlobalBool(RinkebyFlag.Name): + case ctx.Bool(RinkebyFlag.Name): log.Warn("") log.Warn("--------------------------------------------------------------------------------") log.Warn("Please note, Rinkeby has been deprecated. It will still work for the time being,") @@ -1743,25 +1896,25 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { log.Warn("--------------------------------------------------------------------------------") log.Warn("") - if !ctx.GlobalIsSet(NetworkIdFlag.Name) { + if !ctx.IsSet(NetworkIdFlag.Name) { cfg.NetworkId = 4 } cfg.Genesis = core.DefaultRinkebyGenesisBlock() SetDNSDiscoveryDefaults(cfg, params.RinkebyGenesisHash) - case ctx.GlobalBool(GoerliFlag.Name): - if !ctx.GlobalIsSet(NetworkIdFlag.Name) { + case ctx.Bool(GoerliFlag.Name): + if !ctx.IsSet(NetworkIdFlag.Name) { cfg.NetworkId = 5 } cfg.Genesis = core.DefaultGoerliGenesisBlock() SetDNSDiscoveryDefaults(cfg, params.GoerliGenesisHash) - case ctx.GlobalBool(KilnFlag.Name): - if !ctx.GlobalIsSet(NetworkIdFlag.Name) { + case ctx.Bool(KilnFlag.Name): + if !ctx.IsSet(NetworkIdFlag.Name) { cfg.NetworkId = 1337802 } cfg.Genesis = core.DefaultKilnGenesisBlock() SetDNSDiscoveryDefaults(cfg, params.KilnGenesisHash) - case ctx.GlobalBool(DeveloperFlag.Name): - if !ctx.GlobalIsSet(NetworkIdFlag.Name) { + case ctx.Bool(DeveloperFlag.Name): + if !ctx.IsSet(NetworkIdFlag.Name) { cfg.NetworkId = 1337 } cfg.SyncMode = downloader.FullSync @@ -1794,8 +1947,8 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { log.Info("Using developer account", "address", developer.Address) // Create a new developer genesis block or reuse existing one - cfg.Genesis = core.DeveloperGenesisBlock(uint64(ctx.GlobalInt(DeveloperPeriodFlag.Name)), ctx.GlobalUint64(DeveloperGasLimitFlag.Name), developer.Address) - if ctx.GlobalIsSet(DataDirFlag.Name) { + cfg.Genesis = core.DeveloperGenesisBlock(uint64(ctx.Int(DeveloperPeriodFlag.Name)), ctx.Uint64(DeveloperGasLimitFlag.Name), developer.Address) + if ctx.IsSet(DataDirFlag.Name) { // If datadir doesn't exist we need to open db in write-mode // so leveldb can create files. readonly := true @@ -1810,7 +1963,7 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { } chaindb.Close() } - if !ctx.GlobalIsSet(MinerGasPriceFlag.Name) { + if !ctx.IsSet(MinerGasPriceFlag.Name) { cfg.Miner.GasPrice = big.NewInt(1) } default: @@ -1846,10 +1999,8 @@ func RegisterEthService(stack *node.Node, cfg *ethconfig.Config) (ethapi.Backend Fatalf("Failed to register the Ethereum service: %v", err) } stack.RegisterAPIs(tracers.APIs(backend.ApiBackend)) - if backend.BlockChain().Config().TerminalTotalDifficulty != nil { - if err := lescatalyst.Register(stack, backend); err != nil { - Fatalf("Failed to register the catalyst service: %v", err) - } + if err := lescatalyst.Register(stack, backend); err != nil { + Fatalf("Failed to register the Engine API service: %v", err) } return backend.ApiBackend, nil } @@ -1863,48 +2014,59 @@ func RegisterEthService(stack *node.Node, cfg *ethconfig.Config) (ethapi.Backend Fatalf("Failed to create the LES server: %v", err) } } - if backend.BlockChain().Config().TerminalTotalDifficulty != nil { - if err := ethcatalyst.Register(stack, backend); err != nil { - Fatalf("Failed to register the catalyst service: %v", err) - } + if err := ethcatalyst.Register(stack, backend); err != nil { + Fatalf("Failed to register the Engine API service: %v", err) } stack.RegisterAPIs(tracers.APIs(backend.APIBackend)) return backend.APIBackend, backend } -// RegisterEthStatsService configures the Ethereum Stats daemon and adds it to -// the given node. +// RegisterEthStatsService configures the Ethereum Stats daemon and adds it to the node. func RegisterEthStatsService(stack *node.Node, backend ethapi.Backend, url string) { if err := ethstats.New(stack, backend, backend.Engine(), url); err != nil { Fatalf("Failed to register the Ethereum Stats service: %v", err) } } -// RegisterGraphQLService is a utility function to construct a new service and register it against a node. -func RegisterGraphQLService(stack *node.Node, backend ethapi.Backend, cfg node.Config) { - if err := graphql.New(stack, backend, cfg.GraphQLCors, cfg.GraphQLVirtualHosts); err != nil { +// RegisterGraphQLService adds the GraphQL API to the node. +func RegisterGraphQLService(stack *node.Node, backend ethapi.Backend, filterSystem *filters.FilterSystem, cfg *node.Config) { + err := graphql.New(stack, backend, filterSystem, cfg.GraphQLCors, cfg.GraphQLVirtualHosts) + if err != nil { Fatalf("Failed to register the GraphQL service: %v", err) } } +// RegisterFilterAPI adds the eth log filtering RPC API to the node. +func RegisterFilterAPI(stack *node.Node, backend ethapi.Backend, ethcfg *ethconfig.Config) *filters.FilterSystem { + isLightClient := ethcfg.SyncMode == downloader.LightSync + filterSystem := filters.NewFilterSystem(backend, filters.Config{ + LogCacheSize: ethcfg.FilterLogCacheSize, + }) + stack.RegisterAPIs([]rpc.API{{ + Namespace: "eth", + Service: filters.NewFilterAPI(filterSystem, isLightClient), + }}) + return filterSystem +} + func SetupMetrics(ctx *cli.Context) { if metrics.Enabled { log.Info("Enabling metrics collection") var ( - enableExport = ctx.GlobalBool(MetricsEnableInfluxDBFlag.Name) - enableExportV2 = ctx.GlobalBool(MetricsEnableInfluxDBV2Flag.Name) + enableExport = ctx.Bool(MetricsEnableInfluxDBFlag.Name) + enableExportV2 = ctx.Bool(MetricsEnableInfluxDBV2Flag.Name) ) if enableExport || enableExportV2 { CheckExclusive(ctx, MetricsEnableInfluxDBFlag, MetricsEnableInfluxDBV2Flag) - v1FlagIsSet := ctx.GlobalIsSet(MetricsInfluxDBUsernameFlag.Name) || - ctx.GlobalIsSet(MetricsInfluxDBPasswordFlag.Name) + v1FlagIsSet := ctx.IsSet(MetricsInfluxDBUsernameFlag.Name) || + ctx.IsSet(MetricsInfluxDBPasswordFlag.Name) - v2FlagIsSet := ctx.GlobalIsSet(MetricsInfluxDBTokenFlag.Name) || - ctx.GlobalIsSet(MetricsInfluxDBOrganizationFlag.Name) || - ctx.GlobalIsSet(MetricsInfluxDBBucketFlag.Name) + v2FlagIsSet := ctx.IsSet(MetricsInfluxDBTokenFlag.Name) || + ctx.IsSet(MetricsInfluxDBOrganizationFlag.Name) || + ctx.IsSet(MetricsInfluxDBBucketFlag.Name) if enableExport && v2FlagIsSet { Fatalf("Flags --influxdb.metrics.organization, --influxdb.metrics.token, --influxdb.metrics.bucket are only available for influxdb-v2") @@ -1914,32 +2076,32 @@ func SetupMetrics(ctx *cli.Context) { } var ( - endpoint = ctx.GlobalString(MetricsInfluxDBEndpointFlag.Name) - database = ctx.GlobalString(MetricsInfluxDBDatabaseFlag.Name) - username = ctx.GlobalString(MetricsInfluxDBUsernameFlag.Name) - password = ctx.GlobalString(MetricsInfluxDBPasswordFlag.Name) - - token = ctx.GlobalString(MetricsInfluxDBTokenFlag.Name) - bucket = ctx.GlobalString(MetricsInfluxDBBucketFlag.Name) - organization = ctx.GlobalString(MetricsInfluxDBOrganizationFlag.Name) + endpoint = ctx.String(MetricsInfluxDBEndpointFlag.Name) + database = ctx.String(MetricsInfluxDBDatabaseFlag.Name) + username = ctx.String(MetricsInfluxDBUsernameFlag.Name) + password = ctx.String(MetricsInfluxDBPasswordFlag.Name) + + token = ctx.String(MetricsInfluxDBTokenFlag.Name) + bucket = ctx.String(MetricsInfluxDBBucketFlag.Name) + organization = ctx.String(MetricsInfluxDBOrganizationFlag.Name) ) if enableExport { - tagsMap := SplitTagsFlag(ctx.GlobalString(MetricsInfluxDBTagsFlag.Name)) + tagsMap := SplitTagsFlag(ctx.String(MetricsInfluxDBTagsFlag.Name)) log.Info("Enabling metrics export to InfluxDB") go influxdb.InfluxDBWithTags(metrics.DefaultRegistry, 10*time.Second, endpoint, database, username, password, "geth.", tagsMap) } else if enableExportV2 { - tagsMap := SplitTagsFlag(ctx.GlobalString(MetricsInfluxDBTagsFlag.Name)) + tagsMap := SplitTagsFlag(ctx.String(MetricsInfluxDBTagsFlag.Name)) log.Info("Enabling metrics export to InfluxDB (v2)") go influxdb.InfluxDBV2WithTags(metrics.DefaultRegistry, 10*time.Second, endpoint, token, bucket, organization, "geth.", tagsMap) } - if ctx.GlobalIsSet(MetricsHTTPFlag.Name) { - address := fmt.Sprintf("%s:%d", ctx.GlobalString(MetricsHTTPFlag.Name), ctx.GlobalInt(MetricsPortFlag.Name)) + if ctx.IsSet(MetricsHTTPFlag.Name) { + address := fmt.Sprintf("%s:%d", ctx.String(MetricsHTTPFlag.Name), ctx.Int(MetricsPortFlag.Name)) log.Info("Enabling stand-alone metrics HTTP endpoint", "address", address) exp.Setup(address) } @@ -1966,20 +2128,24 @@ func SplitTagsFlag(tagsFlag string) map[string]string { // MakeChainDatabase open an LevelDB using the flags passed to the client and will hard crash if it fails. func MakeChainDatabase(ctx *cli.Context, stack *node.Node, readonly bool) ethdb.Database { var ( - cache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheDatabaseFlag.Name) / 100 - handles = MakeDatabaseHandles(ctx.GlobalInt(FDLimitFlag.Name)) + cache = ctx.Int(CacheFlag.Name) * ctx.Int(CacheDatabaseFlag.Name) / 100 + handles = MakeDatabaseHandles(ctx.Int(FDLimitFlag.Name)) err error chainDb ethdb.Database ) switch { - case ctx.GlobalIsSet(RemoteDBFlag.Name): - log.Info("Using remote db", "url", ctx.GlobalString(RemoteDBFlag.Name)) - chainDb, err = remotedb.New(ctx.GlobalString(RemoteDBFlag.Name)) - case ctx.GlobalString(SyncModeFlag.Name) == "light": + case ctx.IsSet(RemoteDBFlag.Name): + log.Info("Using remote db", "url", ctx.String(RemoteDBFlag.Name), "headers", len(ctx.StringSlice(HttpHeaderFlag.Name))) + client, err := DialRPCWithHeaders(ctx.String(RemoteDBFlag.Name), ctx.StringSlice(HttpHeaderFlag.Name)) + if err != nil { + break + } + chainDb = remotedb.New(client) + case ctx.String(SyncModeFlag.Name) == "light": chainDb, err = stack.OpenDatabase("lightchaindata", cache, handles, "", readonly) default: - chainDb, err = stack.OpenDatabaseWithFreezer("chaindata", cache, handles, ctx.GlobalString(AncientFlag.Name), "", readonly) + chainDb, err = stack.OpenDatabaseWithFreezer("chaindata", cache, handles, ctx.String(AncientFlag.Name), "", readonly) } if err != nil { Fatalf("Could not open database: %v", err) @@ -1987,72 +2153,108 @@ func MakeChainDatabase(ctx *cli.Context, stack *node.Node, readonly bool) ethdb. return chainDb } +func IsNetworkPreset(ctx *cli.Context) bool { + for _, flag := range NetworkFlags { + bFlag, _ := flag.(*cli.BoolFlag) + if ctx.IsSet(bFlag.Name) { + return true + } + } + return false +} + +func DialRPCWithHeaders(endpoint string, headers []string) (*rpc.Client, error) { + if endpoint == "" { + return nil, errors.New("endpoint must be specified") + } + if strings.HasPrefix(endpoint, "rpc:") || strings.HasPrefix(endpoint, "ipc:") { + // Backwards compatibility with geth < 1.5 which required + // these prefixes. + endpoint = endpoint[4:] + } + var opts []rpc.ClientOption + if len(headers) > 0 { + var customHeaders = make(http.Header) + for _, h := range headers { + kv := strings.Split(h, ":") + if len(kv) != 2 { + return nil, fmt.Errorf("invalid http header directive: %q", h) + } + customHeaders.Add(kv[0], kv[1]) + } + opts = append(opts, rpc.WithHeaders(customHeaders)) + } + return rpc.DialOptions(context.Background(), endpoint, opts...) +} + func MakeGenesis(ctx *cli.Context) *core.Genesis { var genesis *core.Genesis switch { - case ctx.GlobalBool(MainnetFlag.Name): + case ctx.Bool(MainnetFlag.Name): genesis = core.DefaultGenesisBlock() - case ctx.GlobalBool(RopstenFlag.Name): + case ctx.Bool(RopstenFlag.Name): genesis = core.DefaultRopstenGenesisBlock() - case ctx.GlobalBool(SepoliaFlag.Name): + case ctx.Bool(SepoliaFlag.Name): genesis = core.DefaultSepoliaGenesisBlock() - case ctx.GlobalBool(RinkebyFlag.Name): + case ctx.Bool(RinkebyFlag.Name): genesis = core.DefaultRinkebyGenesisBlock() - case ctx.GlobalBool(GoerliFlag.Name): + case ctx.Bool(GoerliFlag.Name): genesis = core.DefaultGoerliGenesisBlock() - case ctx.GlobalBool(KilnFlag.Name): + case ctx.Bool(KilnFlag.Name): genesis = core.DefaultKilnGenesisBlock() - case ctx.GlobalBool(DeveloperFlag.Name): + case ctx.Bool(DeveloperFlag.Name): Fatalf("Developer chains are ephemeral") } return genesis } // MakeChain creates a chain manager from set command line flags. -func MakeChain(ctx *cli.Context, stack *node.Node) (chain *core.BlockChain, chainDb ethdb.Database) { - var err error - chainDb = MakeChainDatabase(ctx, stack, false) // TODO(rjl493456442) support read-only database - config, _, err := core.SetupGenesisBlock(chainDb, MakeGenesis(ctx)) +func MakeChain(ctx *cli.Context, stack *node.Node) (*core.BlockChain, ethdb.Database) { + var ( + gspec = MakeGenesis(ctx) + chainDb = MakeChainDatabase(ctx, stack, false) // TODO(rjl493456442) support read-only database + ) + cliqueConfig, err := core.LoadCliqueConfig(chainDb, gspec) if err != nil { Fatalf("%v", err) } - - var engine consensus.Engine - ethashConf := ethconfig.Defaults.Ethash - if ctx.GlobalBool(FakePoWFlag.Name) { - ethashConf.PowMode = ethash.ModeFake + ethashConfig := ethconfig.Defaults.Ethash + if ctx.Bool(FakePoWFlag.Name) { + ethashConfig.PowMode = ethash.ModeFake } - engine = ethconfig.CreateConsensusEngine(stack, config, ðashConf, nil, false, chainDb) - if gcmode := ctx.GlobalString(GCModeFlag.Name); gcmode != "full" && gcmode != "archive" { + engine := ethconfig.CreateConsensusEngine(stack, ðashConfig, cliqueConfig, nil, false, chainDb) + if gcmode := ctx.String(GCModeFlag.Name); gcmode != "full" && gcmode != "archive" { Fatalf("--%s must be either 'full' or 'archive'", GCModeFlag.Name) } cache := &core.CacheConfig{ TrieCleanLimit: ethconfig.Defaults.TrieCleanCache, - TrieCleanNoPrefetch: ctx.GlobalBool(CacheNoPrefetchFlag.Name), + TrieCleanNoPrefetch: ctx.Bool(CacheNoPrefetchFlag.Name), TrieDirtyLimit: ethconfig.Defaults.TrieDirtyCache, - TrieDirtyDisabled: ctx.GlobalString(GCModeFlag.Name) == "archive", + TrieDirtyDisabled: ctx.String(GCModeFlag.Name) == "archive", TrieTimeLimit: ethconfig.Defaults.TrieTimeout, SnapshotLimit: ethconfig.Defaults.SnapshotCache, - Preimages: ctx.GlobalBool(CachePreimagesFlag.Name), + Preimages: ctx.Bool(CachePreimagesFlag.Name), } if cache.TrieDirtyDisabled && !cache.Preimages { cache.Preimages = true log.Info("Enabling recording of key preimages since archive mode is used") } - if !ctx.GlobalBool(SnapshotFlag.Name) { + if !ctx.Bool(SnapshotFlag.Name) { cache.SnapshotLimit = 0 // Disabled } - if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheTrieFlag.Name) { - cache.TrieCleanLimit = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheTrieFlag.Name) / 100 + // Disable snapshot generation/wiping by default + cache.SnapshotNoBuild = true + + if ctx.IsSet(CacheFlag.Name) || ctx.IsSet(CacheTrieFlag.Name) { + cache.TrieCleanLimit = ctx.Int(CacheFlag.Name) * ctx.Int(CacheTrieFlag.Name) / 100 } - if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheGCFlag.Name) { - cache.TrieDirtyLimit = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100 + if ctx.IsSet(CacheFlag.Name) || ctx.IsSet(CacheGCFlag.Name) { + cache.TrieDirtyLimit = ctx.Int(CacheFlag.Name) * ctx.Int(CacheGCFlag.Name) / 100 } - vmcfg := vm.Config{EnablePreimageRecording: ctx.GlobalBool(VMEnableDebugFlag.Name)} + vmcfg := vm.Config{EnablePreimageRecording: ctx.Bool(VMEnableDebugFlag.Name)} - // TODO(rjl493456442) disable snapshot generation/wiping if the chain is read only. // Disable transaction indexing/unindexing by default. - chain, err = core.NewBlockChain(chainDb, cache, config, engine, vmcfg, nil, nil) + chain, err := core.NewBlockChain(chainDb, cache, gspec, nil, engine, vmcfg, nil, nil) if err != nil { Fatalf("Can't create BlockChain: %v", err) } @@ -2063,38 +2265,14 @@ func MakeChain(ctx *cli.Context, stack *node.Node) (chain *core.BlockChain, chai // scripts to preload before starting. func MakeConsolePreloads(ctx *cli.Context) []string { // Skip preloading if there's nothing to preload - if ctx.GlobalString(PreloadJSFlag.Name) == "" { + if ctx.String(PreloadJSFlag.Name) == "" { return nil } // Otherwise resolve absolute paths and return them var preloads []string - for _, file := range strings.Split(ctx.GlobalString(PreloadJSFlag.Name), ",") { + for _, file := range strings.Split(ctx.String(PreloadJSFlag.Name), ",") { preloads = append(preloads, strings.TrimSpace(file)) } return preloads } - -// MigrateFlags sets the global flag from a local flag when it's set. -// This is a temporary function used for migrating old command/flags to the -// new format. -// -// e.g. geth account new --keystore /tmp/mykeystore --lightkdf -// -// is equivalent after calling this method with: -// -// geth --keystore /tmp/mykeystore --lightkdf account new -// -// This allows the use of the existing configuration functionality. -// When all flags are migrated this function can be removed and the existing -// configuration functionality must be changed that is uses local flags -func MigrateFlags(action func(ctx *cli.Context) error) func(*cli.Context) error { - return func(ctx *cli.Context) error { - for _, name := range ctx.FlagNames() { - if ctx.IsSet(name) { - ctx.GlobalSet(name, ctx.String(name)) - } - } - return action(ctx) - } -} diff --git a/cmd/utils/flags_legacy.go b/cmd/utils/flags_legacy.go index a0f64f609b77a..930b68fb91d01 100644 --- a/cmd/utils/flags_legacy.go +++ b/cmd/utils/flags_legacy.go @@ -19,40 +19,33 @@ package utils import ( "fmt" - "github.com/ethereum/go-ethereum/eth/ethconfig" - "gopkg.in/urfave/cli.v1" + "github.com/ethereum/go-ethereum/internal/flags" + "github.com/urfave/cli/v2" ) -var ShowDeprecated = cli.Command{ +var ShowDeprecated = &cli.Command{ Action: showDeprecated, Name: "show-deprecated-flags", Usage: "Show flags that have been deprecated", ArgsUsage: " ", - Category: "MISCELLANEOUS COMMANDS", Description: "Show flags that have been deprecated and will soon be removed", } var DeprecatedFlags = []cli.Flag{ - LegacyMinerGasTargetFlag, NoUSBFlag, } var ( // (Deprecated May 2020, shown in aliased flags section) - NoUSBFlag = cli.BoolFlag{ - Name: "nousb", - Usage: "Disables monitoring for and managing USB hardware wallets (deprecated)", - } - // (Deprecated July 2021, shown in aliased flags section) - LegacyMinerGasTargetFlag = cli.Uint64Flag{ - Name: "miner.gastarget", - Usage: "Target gas floor for mined blocks (deprecated)", - Value: ethconfig.Defaults.Miner.GasFloor, + NoUSBFlag = &cli.BoolFlag{ + Name: "nousb", + Usage: "Disables monitoring for and managing USB hardware wallets (deprecated)", + Category: flags.DeprecatedCategory, } ) // showDeprecated displays deprecated flags that will be soon removed from the codebase. -func showDeprecated(*cli.Context) { +func showDeprecated(*cli.Context) error { fmt.Println("--------------------------------------------------------------------") fmt.Println("The following flags are deprecated and will be removed in the future!") fmt.Println("--------------------------------------------------------------------") @@ -61,4 +54,5 @@ func showDeprecated(*cli.Context) { fmt.Println(flag.String()) } fmt.Println() + return nil } diff --git a/common/compiler/solidity.go b/common/compiler/solidity.go index ad8a44aa04ae6..9de94017c2edf 100644 --- a/common/compiler/solidity.go +++ b/common/compiler/solidity.go @@ -66,13 +66,16 @@ func ParseCombinedJSON(combinedJSON []byte, source string, languageVersion strin contracts := make(map[string]*Contract) for name, info := range output.Contracts { // Parse the individual compilation results. - var abi interface{} + var abi, userdoc, devdoc interface{} if err := json.Unmarshal([]byte(info.Abi), &abi); err != nil { return nil, fmt.Errorf("solc: error reading abi definition (%v)", err) } - var userdoc, devdoc interface{} - json.Unmarshal([]byte(info.Userdoc), &userdoc) - json.Unmarshal([]byte(info.Devdoc), &devdoc) + if err := json.Unmarshal([]byte(info.Userdoc), &userdoc); err != nil { + return nil, fmt.Errorf("solc: error reading userdoc definition (%v)", err) + } + if err := json.Unmarshal([]byte(info.Devdoc), &devdoc); err != nil { + return nil, fmt.Errorf("solc: error reading devdoc definition (%v)", err) + } contracts[name] = &Contract{ Code: "0x" + info.Bin, diff --git a/common/fdlimit/fdlimit_test.go b/common/fdlimit/fdlimit_test.go index 21362b8463a3f..9fd5e9fc3cbd6 100644 --- a/common/fdlimit/fdlimit_test.go +++ b/common/fdlimit/fdlimit_test.go @@ -17,7 +17,6 @@ package fdlimit import ( - "fmt" "testing" ) @@ -30,7 +29,7 @@ func TestFileDescriptorLimits(t *testing.T) { t.Fatal(err) } if hardlimit < target { - t.Skip(fmt.Sprintf("system limit is less than desired test target: %d < %d", hardlimit, target)) + t.Skipf("system limit is less than desired test target: %d < %d", hardlimit, target) } if limit, err := Current(); err != nil || limit <= 0 { diff --git a/common/format.go b/common/format.go index 6fc21af719236..7af41f52d5401 100644 --- a/common/format.go +++ b/common/format.go @@ -27,12 +27,12 @@ import ( // the unnecessary precision off from the formatted textual representation. type PrettyDuration time.Duration -var prettyDurationRe = regexp.MustCompile(`\.[0-9]+`) +var prettyDurationRe = regexp.MustCompile(`\.[0-9]{4,}`) // String implements the Stringer interface, allowing pretty printing of duration // values rounded to three decimals. func (d PrettyDuration) String() string { - label := fmt.Sprintf("%v", time.Duration(d)) + label := time.Duration(d).String() if match := prettyDurationRe.FindString(label); len(match) > 4 { label = strings.Replace(label, match, match[:4], 1) } diff --git a/common/hexutil/hexutil.go b/common/hexutil/hexutil.go index e0241f5f2b075..d3201850a8e4c 100644 --- a/common/hexutil/hexutil.go +++ b/common/hexutil/hexutil.go @@ -18,7 +18,7 @@ Package hexutil implements hex encoding with 0x prefix. This encoding is used by the Ethereum RPC API to transport binary data in JSON payloads. -Encoding Rules +# Encoding Rules All hex data must have prefix "0x". diff --git a/common/math/big.go b/common/math/big.go index 1af5b4d879d66..48427810e1ce7 100644 --- a/common/math/big.go +++ b/common/math/big.go @@ -227,10 +227,10 @@ func U256Bytes(n *big.Int) []byte { // S256 interprets x as a two's complement number. // x must not exceed 256 bits (the result is undefined if it does) and is not modified. // -// S256(0) = 0 -// S256(1) = 1 -// S256(2**255) = -2**255 -// S256(2**256-1) = -1 +// S256(0) = 0 +// S256(1) = 1 +// S256(2**255) = -2**255 +// S256(2**256-1) = -1 func S256(x *big.Int) *big.Int { if x.Cmp(tt255) < 0 { return x diff --git a/common/math/big_test.go b/common/math/big_test.go index f896ec65becfd..803b5e1cc6174 100644 --- a/common/math/big_test.go +++ b/common/math/big_test.go @@ -171,7 +171,6 @@ func BenchmarkByteAt(b *testing.B) { } func BenchmarkByteAtOld(b *testing.B) { - bigint := MustParseBig256("0x18F8F8F1000111000110011100222004330052300000000000000000FEFCF3CC") for i := 0; i < b.N; i++ { PaddedBigBytes(bigint, 32) @@ -244,7 +243,6 @@ func TestBigEndianByteAt(t *testing.T) { if actual != test.exp { t.Fatalf("Expected [%v] %v:th byte to be %v, was %v.", test.x, test.y, test.exp, actual) } - } } func TestLittleEndianByteAt(t *testing.T) { @@ -277,7 +275,6 @@ func TestLittleEndianByteAt(t *testing.T) { if actual != test.exp { t.Fatalf("Expected [%v] %v:th byte to be %v, was %v.", test.x, test.y, test.exp, actual) } - } } diff --git a/common/mclock/simclock.go b/common/mclock/simclock.go index 766ca0f8736c0..f5ad3f8bc0aaa 100644 --- a/common/mclock/simclock.go +++ b/common/mclock/simclock.go @@ -58,7 +58,7 @@ func (s *Simulated) Run(d time.Duration) { s.mu.Lock() s.init() - end := s.now + AbsTime(d) + end := s.now.Add(d) var do []func() for len(s.scheduled) > 0 && s.scheduled[0].at <= end { ev := heap.Pop(&s.scheduled).(*simTimer) @@ -134,7 +134,7 @@ func (s *Simulated) AfterFunc(d time.Duration, fn func()) Timer { func (s *Simulated) schedule(d time.Duration, fn func()) *simTimer { s.init() - at := s.now + AbsTime(d) + at := s.now.Add(d) ev := &simTimer{do: fn, at: at, s: s} heap.Push(&s.scheduled, ev) s.cond.Broadcast() diff --git a/common/prque/lazyqueue.go b/common/prque/lazyqueue.go index 37c2f3bd42af5..13ef3ed2cdbf0 100644 --- a/common/prque/lazyqueue.go +++ b/common/prque/lazyqueue.go @@ -26,9 +26,10 @@ import ( // LazyQueue is a priority queue data structure where priorities can change over // time and are only evaluated on demand. // Two callbacks are required: -// - priority evaluates the actual priority of an item -// - maxPriority gives an upper estimate for the priority in any moment between -// now and the given absolute time +// - priority evaluates the actual priority of an item +// - maxPriority gives an upper estimate for the priority in any moment between +// now and the given absolute time +// // If the upper estimate is exceeded then Update should be called for that item. // A global Refresh function should also be called periodically. type LazyQueue struct { @@ -87,13 +88,13 @@ func (q *LazyQueue) Refresh() { // refresh re-evaluates items in the older queue and swaps the two queues func (q *LazyQueue) refresh(now mclock.AbsTime) { - q.maxUntil = now + mclock.AbsTime(q.period) + q.maxUntil = now.Add(q.period) for q.queue[0].Len() != 0 { q.Push(heap.Pop(q.queue[0]).(*item).value) } q.queue[0], q.queue[1] = q.queue[1], q.queue[0] q.indexOffset = 1 - q.indexOffset - q.maxUntil += mclock.AbsTime(q.period) + q.maxUntil = q.maxUntil.Add(q.period) } // Push adds an item to the queue @@ -163,7 +164,7 @@ func (q *LazyQueue) PopItem() interface{} { return i } -// Remove removes removes the item with the given index. +// Remove removes the item with the given index. func (q *LazyQueue) Remove(index int) interface{} { if index < 0 { return nil diff --git a/common/prque/prque.go b/common/prque/prque.go index 54c78b5fc2ba1..fb02e3418c28a 100755 --- a/common/prque/prque.go +++ b/common/prque/prque.go @@ -41,13 +41,13 @@ func (p *Prque) Push(data interface{}, priority int64) { heap.Push(p.cont, &item{data, priority}) } -// Peek returns the value with the greates priority but does not pop it off. +// Peek returns the value with the greatest priority but does not pop it off. func (p *Prque) Peek() (interface{}, int64) { item := p.cont.blocks[0][0] return item.value, item.priority } -// Pops the value with the greates priority off the stack and returns it. +// Pops the value with the greatest priority off the stack and returns it. // Currently no shrinking is done. func (p *Prque) Pop() (interface{}, int64) { item := heap.Pop(p.cont).(*item) diff --git a/common/size_test.go b/common/size_test.go index 0938d483c4bb0..28f053d39f5d5 100644 --- a/common/size_test.go +++ b/common/size_test.go @@ -25,6 +25,8 @@ func TestStorageSizeString(t *testing.T) { size StorageSize str string }{ + {2839274474874, "2.58 TiB"}, + {2458492810, "2.29 GiB"}, {2381273, "2.27 MiB"}, {2192, "2.14 KiB"}, {12, "12.00 B"}, @@ -36,3 +38,22 @@ func TestStorageSizeString(t *testing.T) { } } } + +func TestStorageSizeTerminalString(t *testing.T) { + tests := []struct { + size StorageSize + str string + }{ + {2839274474874, "2.58TiB"}, + {2458492810, "2.29GiB"}, + {2381273, "2.27MiB"}, + {2192, "2.14KiB"}, + {12, "12.00B"}, + } + + for _, test := range tests { + if test.size.TerminalString() != test.str { + t.Errorf("%f: got %q, want %q", float64(test.size), test.size.TerminalString(), test.str) + } + } +} diff --git a/common/types_test.go b/common/types_test.go index 318e985f870b4..94492278d84ac 100644 --- a/common/types_test.go +++ b/common/types_test.go @@ -155,7 +155,6 @@ func BenchmarkAddressHex(b *testing.B) { } func TestMixedcaseAccount_Address(t *testing.T) { - // https://github.com/ethereum/EIPs/blob/master/EIPS/eip-55.md // Note: 0X{checksum_addr} is not valid according to spec above @@ -192,9 +191,7 @@ func TestMixedcaseAccount_Address(t *testing.T) { if err := json.Unmarshal([]byte(r), &r2); err == nil { t.Errorf("Expected failure, input %v", r) } - } - } func TestHash_Scan(t *testing.T) { diff --git a/consensus/beacon/consensus.go b/consensus/beacon/consensus.go index 1fd7deb872fb4..6d108856e6d6c 100644 --- a/consensus/beacon/consensus.go +++ b/consensus/beacon/consensus.go @@ -45,6 +45,7 @@ var ( errTooManyUncles = errors.New("too many uncles") errInvalidNonce = errors.New("invalid nonce") errInvalidUncleHash = errors.New("invalid uncle hash") + errInvalidTimestamp = errors.New("invalid timestamp") ) // Beacon is a consensus engine that combines the eth1 consensus and proof-of-stake @@ -78,7 +79,10 @@ func (beacon *Beacon) Author(header *types.Header) (common.Address, error) { // VerifyHeader checks whether a header conforms to the consensus rules of the // stock Ethereum consensus engine. func (beacon *Beacon) VerifyHeader(chain consensus.ChainHeaderReader, header *types.Header, seal bool) error { - reached, _ := IsTTDReached(chain, header.ParentHash, header.Number.Uint64()-1) + reached, err := IsTTDReached(chain, header.ParentHash, header.Number.Uint64()-1) + if err != nil { + return err + } if !reached { return beacon.ethone.VerifyHeader(chain, header, seal) } @@ -112,10 +116,23 @@ func (beacon *Beacon) VerifyHeaders(chain consensus.ChainHeaderReader, headers [ break } } - // All the headers have passed the transition point, use new rules. + if len(preHeaders) == 0 { + // All the headers are pos headers. Verify that the parent block reached total terminal difficulty. + if reached, err := IsTTDReached(chain, headers[0].ParentHash, headers[0].Number.Uint64()-1); !reached { + // TTD not reached for the first block, mark subsequent with invalid terminal block + if err == nil { + err = consensus.ErrInvalidTerminalBlock + } + results := make(chan error, len(headers)) + for i := 0; i < len(headers); i++ { + results <- err + } + return make(chan struct{}), results + } return beacon.verifyHeaders(chain, headers, nil) } + // The transition point exists in the middle, separate the headers // into two batches and apply different verification rules for them. var ( @@ -130,6 +147,14 @@ func (beacon *Beacon) VerifyHeaders(chain consensus.ChainHeaderReader, headers [ oldDone, oldResult = beacon.ethone.VerifyHeaders(chain, preHeaders, preSeals) newDone, newResult = beacon.verifyHeaders(chain, postHeaders, preHeaders[len(preHeaders)-1]) ) + // Verify that pre-merge headers don't overflow the TTD + if index, err := verifyTerminalPoWBlock(chain, preHeaders); err != nil { + // Mark all subsequent pow headers with the error. + for i := index; i < len(preHeaders); i++ { + errors[i], done[i] = err, true + } + } + // Collect the results for { for ; done[out]; out++ { results <- errors[out] @@ -139,7 +164,9 @@ func (beacon *Beacon) VerifyHeaders(chain consensus.ChainHeaderReader, headers [ } select { case err := <-oldResult: - errors[old], done[old] = err, true + if !done[old] { // skip TTD-verified failures + errors[old], done[old] = err, true + } old++ case err := <-newResult: errors[new], done[new] = err, true @@ -154,6 +181,33 @@ func (beacon *Beacon) VerifyHeaders(chain consensus.ChainHeaderReader, headers [ return abort, results } +// verifyTerminalPoWBlock verifies that the preHeaders conform to the specification +// wrt. their total difficulty. +// It expects: +// - preHeaders to be at least 1 element +// - the parent of the header element to be stored in the chain correctly +// - the preHeaders to have a set difficulty +// - the last element to be the terminal block +func verifyTerminalPoWBlock(chain consensus.ChainHeaderReader, preHeaders []*types.Header) (int, error) { + td := chain.GetTd(preHeaders[0].ParentHash, preHeaders[0].Number.Uint64()-1) + if td == nil { + return 0, consensus.ErrUnknownAncestor + } + td = new(big.Int).Set(td) + // Check that all blocks before the last one are below the TTD + for i, head := range preHeaders { + if td.Cmp(chain.Config().TerminalTotalDifficulty) >= 0 { + return i, consensus.ErrInvalidTerminalBlock + } + td.Add(td, head.Difficulty) + } + // Check that the last block is the terminal block + if td.Cmp(chain.Config().TerminalTotalDifficulty) < 0 { + return len(preHeaders) - 1, consensus.ErrInvalidTerminalBlock + } + return 0, nil +} + // VerifyUncles verifies that the given block's uncles conform to the consensus // rules of the Ethereum consensus engine. func (beacon *Beacon) VerifyUncles(chain consensus.ChainReader, block *types.Block) error { @@ -170,11 +224,12 @@ func (beacon *Beacon) VerifyUncles(chain consensus.ChainReader, block *types.Blo // verifyHeader checks whether a header conforms to the consensus rules of the // stock Ethereum consensus engine. The difference between the beacon and classic is // (a) The following fields are expected to be constants: -// - difficulty is expected to be 0 -// - nonce is expected to be 0 -// - unclehash is expected to be Hash(emptyHeader) +// - difficulty is expected to be 0 +// - nonce is expected to be 0 +// - unclehash is expected to be Hash(emptyHeader) // to be the desired constants -// (b) the timestamp is not verified anymore +// +// (b) we don't verify if a block is in the future anymore // (c) the extradata is limited to 32 bytes func (beacon *Beacon) verifyHeader(chain consensus.ChainHeaderReader, header, parent *types.Header) error { // Ensure that the header's extra-data section is of a reasonable size @@ -188,6 +243,10 @@ func (beacon *Beacon) verifyHeader(chain consensus.ChainHeaderReader, header, pa if header.UncleHash != types.EmptyUncleHash { return errInvalidUncleHash } + // Verify the timestamp + if header.Time <= parent.Time { + return errInvalidTimestamp + } // Verify the block's difficulty to ensure it's the default constant if beaconDifficulty.Cmp(header.Difficulty) != 0 { return fmt.Errorf("invalid difficulty: have %v, want %v", header.Difficulty, beaconDifficulty) diff --git a/consensus/beacon/consensus_test.go b/consensus/beacon/consensus_test.go new file mode 100644 index 0000000000000..09c0b27c42562 --- /dev/null +++ b/consensus/beacon/consensus_test.go @@ -0,0 +1,137 @@ +package beacon + +import ( + "fmt" + "math/big" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/params" +) + +type mockChain struct { + config *params.ChainConfig + tds map[uint64]*big.Int +} + +func newMockChain() *mockChain { + return &mockChain{ + config: new(params.ChainConfig), + tds: make(map[uint64]*big.Int), + } +} + +func (m *mockChain) Config() *params.ChainConfig { + return m.config +} + +func (m *mockChain) CurrentHeader() *types.Header { panic("not implemented") } + +func (m *mockChain) GetHeader(hash common.Hash, number uint64) *types.Header { + panic("not implemented") +} + +func (m *mockChain) GetHeaderByNumber(number uint64) *types.Header { panic("not implemented") } + +func (m *mockChain) GetHeaderByHash(hash common.Hash) *types.Header { panic("not implemented") } + +func (m *mockChain) GetTd(hash common.Hash, number uint64) *big.Int { + num, ok := m.tds[number] + if ok { + return new(big.Int).Set(num) + } + return nil +} + +func TestVerifyTerminalBlock(t *testing.T) { + chain := newMockChain() + chain.tds[0] = big.NewInt(10) + chain.config.TerminalTotalDifficulty = big.NewInt(50) + + tests := []struct { + preHeaders []*types.Header + ttd *big.Int + err error + index int + }{ + // valid ttd + { + preHeaders: []*types.Header{ + {Number: big.NewInt(1), Difficulty: big.NewInt(10)}, + {Number: big.NewInt(2), Difficulty: big.NewInt(10)}, + {Number: big.NewInt(3), Difficulty: big.NewInt(10)}, + {Number: big.NewInt(4), Difficulty: big.NewInt(10)}, + }, + ttd: big.NewInt(50), + }, + // last block doesn't reach ttd + { + preHeaders: []*types.Header{ + {Number: big.NewInt(1), Difficulty: big.NewInt(10)}, + {Number: big.NewInt(2), Difficulty: big.NewInt(10)}, + {Number: big.NewInt(3), Difficulty: big.NewInt(10)}, + {Number: big.NewInt(4), Difficulty: big.NewInt(9)}, + }, + ttd: big.NewInt(50), + err: consensus.ErrInvalidTerminalBlock, + index: 3, + }, + // two blocks reach ttd + { + preHeaders: []*types.Header{ + {Number: big.NewInt(1), Difficulty: big.NewInt(10)}, + {Number: big.NewInt(2), Difficulty: big.NewInt(10)}, + {Number: big.NewInt(3), Difficulty: big.NewInt(20)}, + {Number: big.NewInt(4), Difficulty: big.NewInt(10)}, + }, + ttd: big.NewInt(50), + err: consensus.ErrInvalidTerminalBlock, + index: 3, + }, + // three blocks reach ttd + { + preHeaders: []*types.Header{ + {Number: big.NewInt(1), Difficulty: big.NewInt(10)}, + {Number: big.NewInt(2), Difficulty: big.NewInt(10)}, + {Number: big.NewInt(3), Difficulty: big.NewInt(20)}, + {Number: big.NewInt(4), Difficulty: big.NewInt(10)}, + {Number: big.NewInt(4), Difficulty: big.NewInt(10)}, + }, + ttd: big.NewInt(50), + err: consensus.ErrInvalidTerminalBlock, + index: 3, + }, + // parent reached ttd + { + preHeaders: []*types.Header{ + {Number: big.NewInt(1), Difficulty: big.NewInt(10)}, + }, + ttd: big.NewInt(9), + err: consensus.ErrInvalidTerminalBlock, + index: 0, + }, + // unknown parent + { + preHeaders: []*types.Header{ + {Number: big.NewInt(4), Difficulty: big.NewInt(10)}, + }, + ttd: big.NewInt(9), + err: consensus.ErrUnknownAncestor, + index: 0, + }, + } + + for i, test := range tests { + fmt.Printf("Test: %v\n", i) + chain.config.TerminalTotalDifficulty = test.ttd + index, err := verifyTerminalPoWBlock(chain, test.preHeaders) + if err != test.err { + t.Fatalf("Invalid error encountered, expected %v got %v", test.err, err) + } + if index != test.index { + t.Fatalf("Invalid index, expected %v got %v", test.index, index) + } + } +} diff --git a/consensus/clique/clique.go b/consensus/clique/clique.go index 950300f034868..dcdfb20c6387a 100644 --- a/consensus/clique/clique.go +++ b/consensus/clique/clique.go @@ -697,9 +697,7 @@ func (c *Clique) Close() error { func (c *Clique) APIs(chain consensus.ChainHeaderReader) []rpc.API { return []rpc.API{{ Namespace: "clique", - Version: "1.0", Service: &API{chain: chain, clique: c}, - Public: false, }} } diff --git a/consensus/clique/clique_test.go b/consensus/clique/clique_test.go index 1bd32acd3746c..f213bc8247d63 100644 --- a/consensus/clique/clique_test.go +++ b/consensus/clique/clique_test.go @@ -45,6 +45,7 @@ func TestReimportMirroredState(t *testing.T) { signer = new(types.HomesteadSigner) ) genspec := &core.Genesis{ + Config: params.AllCliqueProtocolChanges, ExtraData: make([]byte, extraVanity+common.AddressLength+extraSeal), Alloc: map[common.Address]core.GenesisAccount{ addr: {Balance: big.NewInt(10000000000000000)}, @@ -52,13 +53,12 @@ func TestReimportMirroredState(t *testing.T) { BaseFee: big.NewInt(params.InitialBaseFee), } copy(genspec.ExtraData[extraVanity:], addr[:]) - genesis := genspec.MustCommit(db) // Generate a batch of blocks, each properly signed - chain, _ := core.NewBlockChain(db, nil, params.AllCliqueProtocolChanges, engine, vm.Config{}, nil, nil) + chain, _ := core.NewBlockChain(rawdb.NewMemoryDatabase(), nil, genspec, nil, engine, vm.Config{}, nil, nil) defer chain.Stop() - blocks, _ := core.GenerateChain(params.AllCliqueProtocolChanges, genesis, engine, db, 3, func(i int, block *core.BlockGen) { + _, blocks, _ := core.GenerateChainWithGenesis(genspec, engine, 3, func(i int, block *core.BlockGen) { // The chain maker doesn't have access to a chain, so the difficulty will be // lets unset (nil). Set it here to the correct value. block.SetDifficulty(diffInTurn) @@ -87,9 +87,7 @@ func TestReimportMirroredState(t *testing.T) { } // Insert the first two blocks and make sure the chain is valid db = rawdb.NewMemoryDatabase() - genspec.MustCommit(db) - - chain, _ = core.NewBlockChain(db, nil, params.AllCliqueProtocolChanges, engine, vm.Config{}, nil, nil) + chain, _ = core.NewBlockChain(db, nil, genspec, nil, engine, vm.Config{}, nil, nil) defer chain.Stop() if _, err := chain.InsertChain(blocks[:2]); err != nil { @@ -102,7 +100,7 @@ func TestReimportMirroredState(t *testing.T) { // Simulate a crash by creating a new chain on top of the database, without // flushing the dirty states out. Insert the last block, triggering a sidechain // reimport. - chain, _ = core.NewBlockChain(db, nil, params.AllCliqueProtocolChanges, engine, vm.Config{}, nil, nil) + chain, _ = core.NewBlockChain(db, nil, genspec, nil, engine, vm.Config{}, nil, nil) defer chain.Stop() if _, err := chain.InsertChain(blocks[2:]); err != nil { diff --git a/consensus/clique/snapshot_test.go b/consensus/clique/snapshot_test.go index 094868ca744dd..4f9222d0927ae 100644 --- a/consensus/clique/snapshot_test.go +++ b/consensus/clique/snapshot_test.go @@ -305,7 +305,7 @@ func TestClique(t *testing.T) { }, { // Ensure that pending votes don't survive authorization status changes. This // corner case can only appear if a signer is quickly added, removed and then - // readded (or the inverse), while one of the original voters dropped. If a + // re-added (or the inverse), while one of the original voters dropped. If a // past vote is left cached in the system somewhere, this will interfere with // the final signer outcome. signers: []string{"A", "B", "C", "D", "E"}, @@ -344,7 +344,7 @@ func TestClique(t *testing.T) { }, failure: errUnauthorizedSigner, }, { - // An authorized signer that signed recenty should not be able to sign again + // An authorized signer that signed recently should not be able to sign again signers: []string{"A", "B"}, votes: []testerVote{ {signer: "A"}, @@ -401,9 +401,6 @@ func TestClique(t *testing.T) { for j, signer := range signers { copy(genesis.ExtraData[extraVanity+j*common.AddressLength:], signer[:]) } - // Create a pristine blockchain with the genesis injected - db := rawdb.NewMemoryDatabase() - genesis.Commit(db) // Assemble a chain of headers from the cast votes config := *params.TestChainConfig @@ -411,10 +408,12 @@ func TestClique(t *testing.T) { Period: 1, Epoch: tt.epoch, } - engine := New(config.Clique, db) + genesis.Config = &config + + engine := New(config.Clique, rawdb.NewMemoryDatabase()) engine.fakeDiff = true - blocks, _ := core.GenerateChain(&config, genesis.ToBlock(db), engine, db, len(tt.votes), func(j int, gen *core.BlockGen) { + _, blocks, _ := core.GenerateChainWithGenesis(genesis, engine, len(tt.votes), func(j int, gen *core.BlockGen) { // Cast the vote contained in this block gen.SetCoinbase(accounts.address(tt.votes[j].voted)) if tt.votes[j].auth { @@ -450,7 +449,7 @@ func TestClique(t *testing.T) { batches[len(batches)-1] = append(batches[len(batches)-1], block) } // Pass all the headers through clique and ensure tallying succeeds - chain, err := core.NewBlockChain(db, nil, &config, engine, vm.Config{}, nil, nil) + chain, err := core.NewBlockChain(rawdb.NewMemoryDatabase(), nil, genesis, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Errorf("test %d: failed to create test chain: %v", i, err) continue diff --git a/consensus/errors.go b/consensus/errors.go index ac5242fb54c53..d508b6580f55b 100644 --- a/consensus/errors.go +++ b/consensus/errors.go @@ -34,4 +34,8 @@ var ( // ErrInvalidNumber is returned if a block's number doesn't equal its parent's // plus one. ErrInvalidNumber = errors.New("invalid block number") + + // ErrInvalidTerminalBlock is returned if a block is invalid wrt. the terminal + // total difficulty. + ErrInvalidTerminalBlock = errors.New("invalid terminal block") ) diff --git a/consensus/ethash/api.go b/consensus/ethash/api.go index f4d3802e0b37c..da3b0751be542 100644 --- a/consensus/ethash/api.go +++ b/consensus/ethash/api.go @@ -34,10 +34,11 @@ type API struct { // GetWork returns a work package for external miner. // // The work package consists of 3 strings: -// result[0] - 32 bytes hex encoded current block header pow-hash -// result[1] - 32 bytes hex encoded seed hash used for DAG -// result[2] - 32 bytes hex encoded boundary condition ("target"), 2^256/difficulty -// result[3] - hex encoded block number +// +// result[0] - 32 bytes hex encoded current block header pow-hash +// result[1] - 32 bytes hex encoded seed hash used for DAG +// result[2] - 32 bytes hex encoded boundary condition ("target"), 2^256/difficulty +// result[3] - hex encoded block number func (api *API) GetWork() ([4]string, error) { if api.ethash.remote == nil { return [4]string{}, errors.New("not supported") diff --git a/consensus/ethash/consensus.go b/consensus/ethash/consensus.go index 6a93fead29db6..1c38b80ea59b7 100644 --- a/consensus/ethash/consensus.go +++ b/consensus/ethash/consensus.go @@ -45,6 +45,11 @@ var ( maxUncles = 2 // Maximum number of uncles allowed in a single block allowedFutureBlockTimeSeconds = int64(15) // Max seconds from current time allowed for blocks, before they're considered future blocks + // calcDifficultyEip5133 is the difficulty adjustment algorithm as specified by EIP 5133. + // It offsets the bomb a total of 11.4M blocks. + // Specification EIP-5133: https://eips.ethereum.org/EIPS/eip-5133 + calcDifficultyEip5133 = makeDifficultyCalculator(big.NewInt(11_400_000)) + // calcDifficultyEip4345 is the difficulty adjustment algorithm as specified by EIP 4345. // It offsets the bomb a total of 10.7M blocks. // Specification EIP-4345: https://eips.ethereum.org/EIPS/eip-4345 @@ -334,6 +339,8 @@ func (ethash *Ethash) CalcDifficulty(chain consensus.ChainHeaderReader, time uin func CalcDifficulty(config *params.ChainConfig, time uint64, parent *types.Header) *big.Int { next := new(big.Int).Add(parent.Number, big1) switch { + case config.IsGrayGlacier(next): + return calcDifficultyEip5133(time, parent) case config.IsArrowGlacier(next): return calcDifficultyEip4345(time, parent) case config.IsLondon(next): diff --git a/consensus/ethash/consensus_test.go b/consensus/ethash/consensus_test.go index bca424af30e36..db997d737e625 100644 --- a/consensus/ethash/consensus_test.go +++ b/consensus/ethash/consensus_test.go @@ -103,7 +103,7 @@ func TestDifficultyCalculators(t *testing.T) { for i := 0; i < 5000; i++ { // 1 to 300 seconds diff var timeDelta = uint64(1 + rand.Uint32()%3000) - diffBig := big.NewInt(0).SetBytes(randSlice(2, 10)) + diffBig := new(big.Int).SetBytes(randSlice(2, 10)) if diffBig.Cmp(params.MinimumDifficulty) < 0 { diffBig.Set(params.MinimumDifficulty) } diff --git a/consensus/ethash/ethash.go b/consensus/ethash/ethash.go index c196ad0621704..dfe00d4b93c06 100644 --- a/consensus/ethash/ethash.go +++ b/consensus/ethash/ethash.go @@ -278,8 +278,11 @@ func (c *cache) generate(dir string, limit int, lock bool, test bool) { // Iterate over all previous instances and delete old ones for ep := int(c.epoch) - limit; ep >= 0; ep-- { seed := seedHash(uint64(ep)*epochLength + 1) - path := filepath.Join(dir, fmt.Sprintf("cache-R%d-%x%s", algorithmRevision, seed[:8], endian)) - os.Remove(path) + path := filepath.Join(dir, fmt.Sprintf("cache-R%d-%x%s*", algorithmRevision, seed[:8], endian)) + files, _ := filepath.Glob(path) // find also the temp files that are generated. + for _, file := range files { + os.Remove(file) + } } }) } @@ -678,15 +681,11 @@ func (ethash *Ethash) APIs(chain consensus.ChainHeaderReader) []rpc.API { return []rpc.API{ { Namespace: "eth", - Version: "1.0", Service: &API{ethash}, - Public: true, }, { Namespace: "ethash", - Version: "1.0", Service: &API{ethash}, - Public: true, }, } } diff --git a/consensus/ethash/sealer.go b/consensus/ethash/sealer.go index 6fa60ef6a8bbd..ec46963900289 100644 --- a/consensus/ethash/sealer.go +++ b/consensus/ethash/sealer.go @@ -339,10 +339,11 @@ func (s *remoteSealer) loop() { // makeWork creates a work package for external miner. // // The work package consists of 3 strings: -// result[0], 32 bytes hex encoded current block header pow-hash -// result[1], 32 bytes hex encoded seed hash used for DAG -// result[2], 32 bytes hex encoded boundary condition ("target"), 2^256/difficulty -// result[3], hex encoded block number +// +// result[0], 32 bytes hex encoded current block header pow-hash +// result[1], 32 bytes hex encoded seed hash used for DAG +// result[2], 32 bytes hex encoded boundary condition ("target"), 2^256/difficulty +// result[3], hex encoded block number func (s *remoteSealer) makeWork(block *types.Block) { hash := s.ethash.SealHash(block.Header()) s.currentWork[0] = hash.Hex() diff --git a/consensus/misc/dao.go b/consensus/misc/dao.go index 36df036f27350..96995616de569 100644 --- a/consensus/misc/dao.go +++ b/consensus/misc/dao.go @@ -40,10 +40,11 @@ var ( // ensure it conforms to DAO hard-fork rules. // // DAO hard-fork extension to the header validity: -// a) if the node is no-fork, do not accept blocks in the [fork, fork+10) range -// with the fork specific extra-data set -// b) if the node is pro-fork, require blocks in the specific range to have the -// unique extra-data set. +// +// - if the node is no-fork, do not accept blocks in the [fork, fork+10) range +// with the fork specific extra-data set. +// - if the node is pro-fork, require blocks in the specific range to have the +// unique extra-data set. func VerifyDAOHeaderExtraData(config *params.ChainConfig, header *types.Header) error { // Short circuit validation if the node doesn't care about the DAO fork if config.DAOForkBlock == nil { diff --git a/consensus/misc/forks.go b/consensus/misc/forks.go index 4a5e7c37e03c4..a6f3303ea6faa 100644 --- a/consensus/misc/forks.go +++ b/consensus/misc/forks.go @@ -35,7 +35,7 @@ func VerifyForkHashes(config *params.ChainConfig, header *types.Header, uncle bo // If the homestead reprice hash is set, validate it if config.EIP150Block != nil && config.EIP150Block.Cmp(header.Number) == 0 { if config.EIP150Hash != (common.Hash{}) && config.EIP150Hash != header.Hash() { - return fmt.Errorf("homestead gas reprice fork: have 0x%x, want 0x%x", header.Hash(), config.EIP150Hash) + return fmt.Errorf("homestead gas reprice fork: have %#x, want %#x", header.Hash(), config.EIP150Hash) } } // All ok, return diff --git a/console/console.go b/console/console.go index 2f61c1d7a4cf2..7b9ed27e15ecc 100644 --- a/console/console.go +++ b/console/console.go @@ -290,7 +290,7 @@ func (c *Console) AutoCompleteInput(line string, pos int) (string, []string, str if len(line) == 0 || pos == 0 { return "", nil, "" } - // Chunck data to relevant part for autocompletion + // Chunk data to relevant part for autocompletion // E.g. in case of nested lines eth.getBalance(eth.coinb start := pos - 1 for ; start > 0; start-- { @@ -407,7 +407,7 @@ func (c *Console) StopInteractive() { } } -// Interactive starts an interactive user session, where in.put is propted from +// Interactive starts an interactive user session, where input is prompted from // the configured user prompter. func (c *Console) Interactive() { var ( @@ -497,7 +497,7 @@ func (c *Console) readLines(input chan<- string, errc chan<- error, prompt <-cha } } -// countIndents returns the number of identations for the given input. +// countIndents returns the number of indentations for the given input. // In case of invalid input such as var a = } the result can be negative. func countIndents(input string) int { var ( @@ -540,11 +540,6 @@ func countIndents(input string) int { return indents } -// Execute runs the JavaScript file specified as the argument. -func (c *Console) Execute(path string) error { - return c.jsre.Exec(path) -} - // Stop cleans up the console and terminates the runtime environment. func (c *Console) Stop(graceful bool) error { c.stopOnce.Do(func() { diff --git a/console/console_test.go b/console/console_test.go index 04ba91d1576ad..35341fcba0b5d 100644 --- a/console/console_test.go +++ b/console/console_test.go @@ -234,19 +234,6 @@ func TestPreload(t *testing.T) { } } -// Tests that JavaScript scripts can be executes from the configured asset path. -func TestExecute(t *testing.T) { - tester := newTester(t, nil) - defer tester.Close(t) - - tester.console.Execute("exec.js") - - tester.console.Evaluate("execed") - if output := tester.output.String(); !strings.Contains(output, "some-executed-string") { - t.Fatalf("execed variable missing: have %s, want %s", output, "some-executed-string") - } -} - // Tests that the JavaScript objects returned by statement executions are properly // pretty printed instead of just displaying "[object]". func TestPrettyPrint(t *testing.T) { diff --git a/console/testdata/exec.js b/console/testdata/exec.js deleted file mode 100644 index 59e34d7c40334..0000000000000 --- a/console/testdata/exec.js +++ /dev/null @@ -1 +0,0 @@ -var execed = "some-executed-string"; diff --git a/core/asm/asm.go b/core/asm/asm.go index f3f129714d312..7c1e14ec01ead 100644 --- a/core/asm/asm.go +++ b/core/asm/asm.go @@ -109,7 +109,7 @@ func PrintDisassembled(code string) error { it := NewInstructionIterator(script) for it.Next() { if it.Arg() != nil && 0 < len(it.Arg()) { - fmt.Printf("%05x: %v 0x%x\n", it.PC(), it.Op(), it.Arg()) + fmt.Printf("%05x: %v %#x\n", it.PC(), it.Op(), it.Arg()) } else { fmt.Printf("%05x: %v\n", it.PC(), it.Op()) } @@ -124,7 +124,7 @@ func Disassemble(script []byte) ([]string, error) { it := NewInstructionIterator(script) for it.Next() { if it.Arg() != nil && 0 < len(it.Arg()) { - instrs = append(instrs, fmt.Sprintf("%05x: %v 0x%x\n", it.PC(), it.Op(), it.Arg())) + instrs = append(instrs, fmt.Sprintf("%05x: %v %#x\n", it.PC(), it.Op(), it.Arg())) } else { instrs = append(instrs, fmt.Sprintf("%05x: %v\n", it.PC(), it.Op())) } diff --git a/core/beacon/types.go b/core/beacon/types.go index 97bf66cd3fe41..e06ab5c692d97 100644 --- a/core/beacon/types.go +++ b/core/beacon/types.go @@ -42,7 +42,7 @@ type payloadAttributesMarshaling struct { //go:generate go run github.com/fjl/gencodec -type ExecutableDataV1 -field-override executableDataMarshaling -out gen_ed.go -// ExecutableDataV1 structure described at https://github.com/ethereum/execution-apis/src/engine/specification.md +// ExecutableDataV1 structure described at https://github.com/ethereum/execution-apis/tree/main/src/engine/specification.md type ExecutableDataV1 struct { ParentHash common.Hash `json:"parentHash" gencodec:"required"` FeeRecipient common.Address `json:"feeRecipient" gencodec:"required"` @@ -136,9 +136,11 @@ func decodeTransactions(enc [][]byte) ([]*types.Transaction, error) { // ExecutableDataToBlock constructs a block from executable data. // It verifies that the following fields: -// len(extraData) <= 32 -// uncleHash = emptyUncleHash -// difficulty = 0 +// +// len(extraData) <= 32 +// uncleHash = emptyUncleHash +// difficulty = 0 +// // and that the blockhash of the constructed block matches the parameters. func ExecutableDataToBlock(params ExecutableDataV1) (*types.Block, error) { txs, err := decodeTransactions(params.Transactions) diff --git a/core/bench_test.go b/core/bench_test.go index 3006e55131716..f7cf0146060dd 100644 --- a/core/bench_test.go +++ b/core/bench_test.go @@ -187,16 +187,15 @@ func benchInsertChain(b *testing.B, disk bool, gen func(int, *BlockGen)) { // Generate a chain of b.N blocks using the supplied block // generator function. - gspec := Genesis{ + gspec := &Genesis{ Config: params.TestChainConfig, Alloc: GenesisAlloc{benchRootAddr: {Balance: benchRootFunds}}, } - genesis := gspec.MustCommit(db) - chain, _ := GenerateChain(gspec.Config, genesis, ethash.NewFaker(), db, b.N, gen) + _, chain, _ := GenerateChainWithGenesis(gspec, ethash.NewFaker(), b.N, gen) // Time the insertion of the new chain. // State and blocks are stored in the same DB. - chainman, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) + chainman, _ := NewBlockChain(db, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) defer chainman.Stop() b.ReportAllocs() b.ResetTimer() @@ -262,6 +261,11 @@ func makeChainForBench(db ethdb.Database, full bool, count uint64) { rawdb.WriteCanonicalHash(db, hash, n) rawdb.WriteTd(db, hash, n, big.NewInt(int64(n+1))) + if n == 0 { + rawdb.WriteChainConfig(db, hash, params.AllEthashProtocolChanges) + } + rawdb.WriteHeadHeaderHash(db, hash) + if full || n == 0 { block := types.NewBlockWithHeader(header) rawdb.WriteBody(db, hash, n, block.Body()) @@ -303,7 +307,7 @@ func benchReadChain(b *testing.B, full bool, count uint64) { if err != nil { b.Fatalf("error opening database at %v: %v", dir, err) } - chain, err := NewBlockChain(db, &cacheConfig, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil) + chain, err := NewBlockChain(db, &cacheConfig, nil, nil, ethash.NewFaker(), vm.Config{}, nil, nil) if err != nil { b.Fatalf("error creating chain: %v", err) } diff --git a/core/block_validator_test.go b/core/block_validator_test.go index 0f183ba527784..3bdb20e7e1e77 100644 --- a/core/block_validator_test.go +++ b/core/block_validator_test.go @@ -39,17 +39,15 @@ import ( func TestHeaderVerification(t *testing.T) { // Create a simple chain to verify var ( - testdb = rawdb.NewMemoryDatabase() - gspec = &Genesis{Config: params.TestChainConfig} - genesis = gspec.MustCommit(testdb) - blocks, _ = GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), testdb, 8, nil) + gspec = &Genesis{Config: params.TestChainConfig} + _, blocks, _ = GenerateChainWithGenesis(gspec, ethash.NewFaker(), 8, nil) ) headers := make([]*types.Header, len(blocks)) for i, block := range blocks { headers[i] = block.Header() } // Run the header checker for blocks one-by-one, checking for both valid and invalid nonces - chain, _ := NewBlockChain(testdb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil) + chain, _ := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) defer chain.Stop() for i := 0; i < len(blocks); i++ { @@ -89,70 +87,66 @@ func TestHeaderVerificationForMergingEthash(t *testing.T) { testHeaderVerificati // Tests the verification for eth1/2 merging, including pre-merge and post-merge func testHeaderVerificationForMerging(t *testing.T, isClique bool) { var ( - testdb = rawdb.NewMemoryDatabase() - preBlocks []*types.Block - postBlocks []*types.Block - runEngine consensus.Engine - chainConfig *params.ChainConfig - merger = consensus.NewMerger(rawdb.NewMemoryDatabase()) + gspec *Genesis + preBlocks []*types.Block + postBlocks []*types.Block + engine consensus.Engine + merger = consensus.NewMerger(rawdb.NewMemoryDatabase()) ) if isClique { var ( key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") addr = crypto.PubkeyToAddress(key.PublicKey) - engine = clique.New(params.AllCliqueProtocolChanges.Clique, testdb) + config = *params.AllCliqueProtocolChanges ) - genspec := &Genesis{ + engine = beacon.New(clique.New(params.AllCliqueProtocolChanges.Clique, rawdb.NewMemoryDatabase())) + gspec = &Genesis{ + Config: &config, ExtraData: make([]byte, 32+common.AddressLength+crypto.SignatureLength), Alloc: map[common.Address]GenesisAccount{ addr: {Balance: big.NewInt(1)}, }, - BaseFee: big.NewInt(params.InitialBaseFee), + BaseFee: big.NewInt(params.InitialBaseFee), + Difficulty: new(big.Int), } - copy(genspec.ExtraData[32:], addr[:]) - genesis := genspec.MustCommit(testdb) + copy(gspec.ExtraData[32:], addr[:]) - genEngine := beacon.New(engine) - preBlocks, _ = GenerateChain(params.AllCliqueProtocolChanges, genesis, genEngine, testdb, 8, nil) td := 0 - for i, block := range preBlocks { + genDb, blocks, _ := GenerateChainWithGenesis(gspec, engine, 8, nil) + for i, block := range blocks { header := block.Header() if i > 0 { - header.ParentHash = preBlocks[i-1].Hash() + header.ParentHash = blocks[i-1].Hash() } header.Extra = make([]byte, 32+crypto.SignatureLength) header.Difficulty = big.NewInt(2) - sig, _ := crypto.Sign(genEngine.SealHash(header).Bytes(), key) + sig, _ := crypto.Sign(engine.SealHash(header).Bytes(), key) copy(header.Extra[len(header.Extra)-crypto.SignatureLength:], sig) - preBlocks[i] = block.WithSeal(header) + blocks[i] = block.WithSeal(header) + // calculate td td += int(block.Difficulty().Uint64()) } - config := *params.AllCliqueProtocolChanges - config.TerminalTotalDifficulty = big.NewInt(int64(td)) - postBlocks, _ = GenerateChain(&config, preBlocks[len(preBlocks)-1], genEngine, testdb, 8, nil) - chainConfig = &config - runEngine = beacon.New(engine) + preBlocks = blocks + gspec.Config.TerminalTotalDifficulty = big.NewInt(int64(td)) + postBlocks, _ = GenerateChain(gspec.Config, preBlocks[len(preBlocks)-1], engine, genDb, 8, nil) } else { - gspec := &Genesis{Config: params.TestChainConfig} - genesis := gspec.MustCommit(testdb) - genEngine := beacon.New(ethash.NewFaker()) + config := *params.TestChainConfig + gspec = &Genesis{Config: &config} + engine = beacon.New(ethash.NewFaker()) - preBlocks, _ = GenerateChain(params.TestChainConfig, genesis, genEngine, testdb, 8, nil) td := 0 + genDb, blocks, _ := GenerateChainWithGenesis(gspec, engine, 8, nil) for _, block := range preBlocks { // calculate td td += int(block.Difficulty().Uint64()) } - config := *params.TestChainConfig - config.TerminalTotalDifficulty = big.NewInt(int64(td)) - postBlocks, _ = GenerateChain(params.TestChainConfig, preBlocks[len(preBlocks)-1], genEngine, testdb, 8, nil) - - chainConfig = &config - runEngine = beacon.New(ethash.NewFaker()) + preBlocks = blocks + gspec.Config.TerminalTotalDifficulty = big.NewInt(int64(td)) + postBlocks, _ = GenerateChain(gspec.Config, preBlocks[len(preBlocks)-1], engine, genDb, 8, nil) } - + // Assemble header batch preHeaders := make([]*types.Header, len(preBlocks)) for i, block := range preBlocks { preHeaders[i] = block.Header() @@ -168,12 +162,12 @@ func testHeaderVerificationForMerging(t *testing.T, isClique bool) { t.Logf("Log header after the merging %d: %v", block.NumberU64(), string(blob)) } // Run the header checker for blocks one-by-one, checking for both valid and invalid nonces - chain, _ := NewBlockChain(testdb, nil, chainConfig, runEngine, vm.Config{}, nil, nil) + chain, _ := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, engine, vm.Config{}, nil, nil) defer chain.Stop() // Verify the blocks before the merging for i := 0; i < len(preBlocks); i++ { - _, results := runEngine.VerifyHeaders(chain, []*types.Header{preHeaders[i]}, []bool{true}) + _, results := engine.VerifyHeaders(chain, []*types.Header{preHeaders[i]}, []bool{true}) // Wait for the verification result select { case result := <-results: @@ -198,7 +192,7 @@ func testHeaderVerificationForMerging(t *testing.T, isClique bool) { // Verify the blocks after the merging for i := 0; i < len(postBlocks); i++ { - _, results := runEngine.VerifyHeaders(chain, []*types.Header{postHeaders[i]}, []bool{true}) + _, results := engine.VerifyHeaders(chain, []*types.Header{postHeaders[i]}, []bool{true}) // Wait for the verification result select { case result := <-results: @@ -230,7 +224,7 @@ func testHeaderVerificationForMerging(t *testing.T, isClique bool) { headers = append(headers, block.Header()) seals = append(seals, true) } - _, results := runEngine.VerifyHeaders(chain, headers, seals) + _, results := engine.VerifyHeaders(chain, headers, seals) for i := 0; i < len(headers); i++ { select { case result := <-results: @@ -257,10 +251,8 @@ func TestHeaderConcurrentVerification32(t *testing.T) { testHeaderConcurrentVeri func testHeaderConcurrentVerification(t *testing.T, threads int) { // Create a simple chain to verify var ( - testdb = rawdb.NewMemoryDatabase() - gspec = &Genesis{Config: params.TestChainConfig} - genesis = gspec.MustCommit(testdb) - blocks, _ = GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), testdb, 8, nil) + gspec = &Genesis{Config: params.TestChainConfig} + _, blocks, _ = GenerateChainWithGenesis(gspec, ethash.NewFaker(), 8, nil) ) headers := make([]*types.Header, len(blocks)) seals := make([]bool, len(blocks)) @@ -279,11 +271,11 @@ func testHeaderConcurrentVerification(t *testing.T, threads int) { var results <-chan error if valid { - chain, _ := NewBlockChain(testdb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil) + chain, _ := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) _, results = chain.engine.VerifyHeaders(chain, headers, seals) chain.Stop() } else { - chain, _ := NewBlockChain(testdb, nil, params.TestChainConfig, ethash.NewFakeFailer(uint64(len(headers)-1)), vm.Config{}, nil, nil) + chain, _ := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, ethash.NewFakeFailer(uint64(len(headers)-1)), vm.Config{}, nil, nil) _, results = chain.engine.VerifyHeaders(chain, headers, seals) chain.Stop() } @@ -329,10 +321,8 @@ func TestHeaderConcurrentAbortion32(t *testing.T) { testHeaderConcurrentAbortion func testHeaderConcurrentAbortion(t *testing.T, threads int) { // Create a simple chain to verify var ( - testdb = rawdb.NewMemoryDatabase() - gspec = &Genesis{Config: params.TestChainConfig} - genesis = gspec.MustCommit(testdb) - blocks, _ = GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), testdb, 1024, nil) + gspec = &Genesis{Config: params.TestChainConfig} + _, blocks, _ = GenerateChainWithGenesis(gspec, ethash.NewFaker(), 1024, nil) ) headers := make([]*types.Header, len(blocks)) seals := make([]bool, len(blocks)) @@ -346,7 +336,7 @@ func testHeaderConcurrentAbortion(t *testing.T, threads int) { defer runtime.GOMAXPROCS(old) // Start the verifications and immediately abort - chain, _ := NewBlockChain(testdb, nil, params.TestChainConfig, ethash.NewFakeDelayer(time.Millisecond), vm.Config{}, nil, nil) + chain, _ := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, ethash.NewFakeDelayer(time.Millisecond), vm.Config{}, nil, nil) defer chain.Stop() abort, results := chain.engine.VerifyHeaders(chain, headers, seals) diff --git a/core/blockchain.go b/core/blockchain.go index b8de2d4844567..2821fedb0f340 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -22,7 +22,9 @@ import ( "fmt" "io" "math/big" + "runtime" "sort" + "strings" "sync" "sync/atomic" "time" @@ -39,6 +41,7 @@ import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/internal/syncx" + "github.com/ethereum/go-ethereum/internal/version" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/params" @@ -51,6 +54,7 @@ var ( headHeaderGauge = metrics.NewRegisteredGauge("chain/head/header", nil) headFastBlockGauge = metrics.NewRegisteredGauge("chain/head/receipt", nil) headFinalizedBlockGauge = metrics.NewRegisteredGauge("chain/head/finalized", nil) + headSafeBlockGauge = metrics.NewRegisteredGauge("chain/head/safe", nil) accountReadTimer = metrics.NewRegisteredTimer("chain/account/reads", nil) accountHashTimer = metrics.NewRegisteredTimer("chain/account/hashes", nil) @@ -66,6 +70,8 @@ var ( snapshotStorageReadTimer = metrics.NewRegisteredTimer("chain/snapshot/storage/reads", nil) snapshotCommitTimer = metrics.NewRegisteredTimer("chain/snapshot/commits", nil) + triedbCommitTimer = metrics.NewRegisteredTimer("chain/triedb/commits", nil) + blockInsertTimer = metrics.NewRegisteredTimer("chain/inserts", nil) blockValidationTimer = metrics.NewRegisteredTimer("chain/validation", nil) blockExecutionTimer = metrics.NewRegisteredTimer("chain/execution", nil) @@ -131,7 +137,8 @@ type CacheConfig struct { SnapshotLimit int // Memory allowance (MB) to use for caching snapshot entries in memory Preimages bool // Whether to store preimage of trie key to the disk - SnapshotWait bool // Wait for snapshot construction on startup. TODO(karalabe): This is a dirty hack for testing, nuke it + SnapshotNoBuild bool // Whether the background generation is allowed + SnapshotWait bool // Wait for snapshot construction on startup. TODO(karalabe): This is a dirty hack for testing, nuke it } // defaultCacheConfig are the default caching values if none are specified by the @@ -191,6 +198,7 @@ type BlockChain struct { currentBlock atomic.Value // Current head of the block chain currentFastBlock atomic.Value // Current head of the fast-sync chain (may be above the block chain!) currentFinalizedBlock atomic.Value // Current finalized head + currentSafeBlock atomic.Value // Current safe head stateCache state.Database // State database to reuse between imports (contains state cache) bodyCache *lru.Cache // Cache for the most recent block bodies @@ -216,7 +224,7 @@ type BlockChain struct { // NewBlockChain returns a fully initialised block chain using information // available in the database. It initialises the default Ethereum Validator // and Processor. -func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *params.ChainConfig, engine consensus.Engine, vmConfig vm.Config, shouldPreserve func(header *types.Header) bool, txLookupLimit *uint64) (*BlockChain, error) { +func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis, overrides *ChainOverrides, engine consensus.Engine, vmConfig vm.Config, shouldPreserve func(header *types.Header) bool, txLookupLimit *uint64) (*BlockChain, error) { if cacheConfig == nil { cacheConfig = defaultCacheConfig } @@ -227,6 +235,21 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par txLookupCache, _ := lru.New(txLookupCacheLimit) futureBlocks, _ := lru.New(maxFutureBlocks) + // Setup the genesis block, commit the provided genesis specification + // to database if the genesis block is not present yet, or load the + // stored one from database. + chainConfig, genesisHash, genesisErr := SetupGenesisBlockWithOverride(db, genesis, overrides) + if _, ok := genesisErr.(*params.ConfigCompatError); genesisErr != nil && !ok { + return nil, genesisErr + } + log.Info("") + log.Info(strings.Repeat("-", 153)) + for _, line := range strings.Split(chainConfig.Description(), "\n") { + log.Info(line) + } + log.Info(strings.Repeat("-", 153)) + log.Info("") + bc := &BlockChain{ chainConfig: chainConfig, cacheConfig: cacheConfig, @@ -267,23 +290,18 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par bc.currentBlock.Store(nilBlock) bc.currentFastBlock.Store(nilBlock) bc.currentFinalizedBlock.Store(nilBlock) + bc.currentSafeBlock.Store(nilBlock) - // Initialize the chain with ancient data if it isn't empty. - var txIndexBlock uint64 - + // If Geth is initialized with an external ancient store, re-initialize the + // missing chain indexes and chain flags. This procedure can survive crash + // and can be resumed in next restart since chain flags are updated in last step. if bc.empty() { rawdb.InitDatabaseFromFreezer(bc.db) - // If ancient database is not empty, reconstruct all missing - // indices in the background. - frozen, _ := bc.db.Ancients() - if frozen > 0 { - txIndexBlock = frozen - } } + // Load blockchain states from disk if err := bc.loadLastState(); err != nil { return nil, err } - // Make sure the state associated with the block is available head := bc.CurrentBlock() if _, err := state.New(head.Root(), bc.stateCache, bc.snaps); err != nil { @@ -378,21 +396,19 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par log.Warn("Enabling snapshot recovery", "chainhead", head.NumberU64(), "diskbase", *layer) recover = true } - bc.snaps, _ = snapshot.New(bc.db, bc.stateCache.TrieDB(), bc.cacheConfig.SnapshotLimit, head.Root(), !bc.cacheConfig.SnapshotWait, true, recover) + snapconfig := snapshot.Config{ + CacheSize: bc.cacheConfig.SnapshotLimit, + Recovery: recover, + NoBuild: bc.cacheConfig.SnapshotNoBuild, + AsyncBuild: !bc.cacheConfig.SnapshotWait, + } + bc.snaps, _ = snapshot.New(snapconfig, bc.db, bc.stateCache.TrieDB(), head.Root()) } // Start future block processor. bc.wg.Add(1) go bc.updateFutureBlocks() - // Start tx indexer/unindexer. - if txLookupLimit != nil { - bc.txLookupLimit = *txLookupLimit - - bc.wg.Add(1) - go bc.maintainTxIndex(txIndexBlock) - } - // If periodic cache journal is required, spin it up. if bc.cacheConfig.TrieCleanRejournal > 0 { if bc.cacheConfig.TrieCleanRejournal < time.Minute { @@ -406,6 +422,19 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par triedb.SaveCachePeriodically(bc.cacheConfig.TrieCleanJournal, bc.cacheConfig.TrieCleanRejournal, bc.quit) }() } + // Rewind the chain in case of an incompatible config upgrade. + if compat, ok := genesisErr.(*params.ConfigCompatError); ok { + log.Warn("Rewinding chain to upgrade configuration", "err", compat) + bc.SetHead(compat.RewindTo) + rawdb.WriteChainConfig(db, genesisHash, chainConfig) + } + // Start tx indexer/unindexer if required. + if txLookupLimit != nil { + bc.txLookupLimit = *txLookupLimit + + bc.wg.Add(1) + go bc.maintainTxIndex() + } return bc, nil } @@ -464,11 +493,15 @@ func (bc *BlockChain) loadLastState() error { } } - // Restore the last known finalized block + // Restore the last known finalized block and safe block + // Note: the safe block is not stored on disk and it is set to the last + // known finalized block on startup if head := rawdb.ReadFinalizedBlockHash(bc.db); head != (common.Hash{}) { if block := bc.GetBlockByHash(head); block != nil { bc.currentFinalizedBlock.Store(block) headFinalizedBlockGauge.Update(int64(block.NumberU64())) + bc.currentSafeBlock.Store(block) + headSafeBlockGauge.Update(int64(block.NumberU64())) } } // Issue a status log for the user @@ -504,8 +537,23 @@ func (bc *BlockChain) SetHead(head uint64) error { // SetFinalized sets the finalized block. func (bc *BlockChain) SetFinalized(block *types.Block) { bc.currentFinalizedBlock.Store(block) - rawdb.WriteFinalizedBlockHash(bc.db, block.Hash()) - headFinalizedBlockGauge.Update(int64(block.NumberU64())) + if block != nil { + rawdb.WriteFinalizedBlockHash(bc.db, block.Hash()) + headFinalizedBlockGauge.Update(int64(block.NumberU64())) + } else { + rawdb.WriteFinalizedBlockHash(bc.db, common.Hash{}) + headFinalizedBlockGauge.Update(0) + } +} + +// SetSafe sets the safe block. +func (bc *BlockChain) SetSafe(block *types.Block) { + bc.currentSafeBlock.Store(block) + if block != nil { + headSafeBlockGauge.Update(int64(block.NumberU64())) + } else { + headSafeBlockGauge.Update(0) + } } // setHeadBeyondRoot rewinds the local chain to a new head with the extra condition @@ -663,6 +711,16 @@ func (bc *BlockChain) setHeadBeyondRoot(head uint64, root common.Hash, repair bo bc.txLookupCache.Purge() bc.futureBlocks.Purge() + // Clear safe block, finalized block if needed + if safe := bc.CurrentSafeBlock(); safe != nil && head < safe.NumberU64() { + log.Warn("SetHead invalidated safe block") + bc.SetSafe(nil) + } + if finalized := bc.CurrentFinalizedBlock(); finalized != nil && head < finalized.NumberU64() { + log.Error("SetHead invalidated finalized block") + bc.SetFinalized(nil) + } + return rootNumber, bc.loadLastState() } @@ -674,10 +732,10 @@ func (bc *BlockChain) SnapSyncCommitHead(hash common.Hash) error { if block == nil { return fmt.Errorf("non existent block [%x..]", hash[:4]) } - if _, err := trie.NewSecure(block.Root(), bc.stateCache.TrieDB()); err != nil { - return err + root := block.Root() + if !bc.HasState(root) { + return fmt.Errorf("non existent state [%x..]", root[:4]) } - // If all checks out, manually set the head block. if !bc.chainmu.TryLock() { return errChainStopped @@ -689,7 +747,7 @@ func (bc *BlockChain) SnapSyncCommitHead(hash common.Hash) error { // Destroy any existing state snapshot and regenerate it in the background, // also resuming the normal maintenance of any previously paused snapshot. if bc.snaps != nil { - bc.snaps.Rebuild(block.Root()) + bc.snaps.Rebuild(root) } log.Info("Committed new head block", "number", block.Number(), "hash", hash) return nil @@ -739,22 +797,25 @@ func (bc *BlockChain) Export(w io.Writer) error { // ExportN writes a subset of the active chain to the given writer. func (bc *BlockChain) ExportN(w io.Writer, first uint64, last uint64) error { - if !bc.chainmu.TryLock() { - return errChainStopped - } - defer bc.chainmu.Unlock() - if first > last { return fmt.Errorf("export failed: first (%d) is greater than last (%d)", first, last) } log.Info("Exporting batch of blocks", "count", last-first+1) - start, reported := time.Now(), time.Now() + var ( + parentHash common.Hash + start = time.Now() + reported = time.Now() + ) for nr := first; nr <= last; nr++ { block := bc.GetBlockByNumber(nr) if block == nil { return fmt.Errorf("export failed on #%d: not found", nr) } + if nr > first && block.ParentHash() != parentHash { + return fmt.Errorf("export failed: chain reorg during export") + } + parentHash = block.Hash() if err := block.EncodeRLP(w); err != nil { return err } @@ -858,6 +919,10 @@ func (bc *BlockChain) Stop() { log.Error("Dangling trie nodes after full cleanup") } } + // Flush the collected preimages to disk + if err := bc.stateCache.TrieDB().CommitPreimages(); err != nil { + log.Error("Failed to commit trie preimages", "err", err) + } // Ensure all live cached entries be saved into disk, so that we can skip // cache warmup when node restarts. if bc.cacheConfig.TrieCleanJournal != "" { @@ -1209,7 +1274,7 @@ func (bc *BlockChain) writeKnownBlock(block *types.Block) error { // writeBlockWithState writes block, metadata and corresponding state data to the // database. -func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types.Receipt, logs []*types.Log, state *state.StateDB) error { +func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types.Receipt, state *state.StateDB) error { // Calculate the total difficulty of the block ptd := bc.GetTd(block.ParentHash(), block.NumberU64()-1) if ptd == nil { @@ -1304,7 +1369,7 @@ func (bc *BlockChain) WriteBlockAndSetHead(block *types.Block, receipts []*types // writeBlockAndSetHead is the internal implementation of WriteBlockAndSetHead. // This function expects the chain mutex to be held. func (bc *BlockChain) writeBlockAndSetHead(block *types.Block, receipts []*types.Receipt, logs []*types.Log, state *state.StateDB, emitHeadEvent bool) (status WriteStatus, err error) { - if err := bc.writeBlockWithState(block, receipts, logs, state); err != nil { + if err := bc.writeBlockWithState(block, receipts, state); err != nil { return NonStatTy, err } currentBlock := bc.CurrentBlock() @@ -1336,7 +1401,7 @@ func (bc *BlockChain) writeBlockAndSetHead(block *types.Block, receipts []*types } // In theory we should fire a ChainHeadEvent when we inject // a canonical block, but sometimes we can insert a batch of - // canonicial blocks. Avoid firing too many ChainHeadEvents, + // canonical blocks. Avoid firing too many ChainHeadEvents, // we will fire an accumulated ChainHeadEvent and disable fire // event here. if emitHeadEvent { @@ -1573,7 +1638,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals, setHead bool) // block in the middle. It can only happen in the clique chain. Whenever // we insert blocks via `insertSideChain`, we only commit `td`, `header` // and `body` if it's non-existent. Since we don't have receipts without - // reexecution, so nothing to commit. But if the sidechain will be adpoted + // reexecution, so nothing to commit. But if the sidechain will be adopted // as the canonical chain eventually, it needs to be reexecuted for missing // state, but if it's this special case here(skip reexecution) we will lose // the empty receipt entry. @@ -1668,7 +1733,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals, setHead bool) var status WriteStatus if !setHead { // Don't set the head, only insert the block - err = bc.writeBlockWithState(block, receipts, logs, statedb) + err = bc.writeBlockWithState(block, receipts, statedb) } else { status, err = bc.writeBlockAndSetHead(block, receipts, logs, statedb, false) } @@ -1680,8 +1745,9 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals, setHead bool) accountCommitTimer.Update(statedb.AccountCommits) // Account commits are complete, we can mark them storageCommitTimer.Update(statedb.StorageCommits) // Storage commits are complete, we can mark them snapshotCommitTimer.Update(statedb.SnapshotCommits) // Snapshot commits are complete, we can mark them + triedbCommitTimer.Update(statedb.TrieDBCommits) // Triedb commits are complete, we can mark them - blockWriteTimer.Update(time.Since(substart) - statedb.AccountCommits - statedb.StorageCommits - statedb.SnapshotCommits) + blockWriteTimer.Update(time.Since(substart) - statedb.AccountCommits - statedb.StorageCommits - statedb.SnapshotCommits - statedb.TrieDBCommits) blockInsertTimer.UpdateSince(start) // Report the import stats before returning the various results @@ -1939,21 +2005,6 @@ func (bc *BlockChain) collectLogs(hash common.Hash, removed bool) []*types.Log { return logs } -// mergeLogs returns a merged log slice with specified sort order. -func mergeLogs(logs [][]*types.Log, reverse bool) []*types.Log { - var ret []*types.Log - if reverse { - for i := len(logs) - 1; i >= 0; i-- { - ret = append(ret, logs[i]...) - } - } else { - for i := 0; i < len(logs); i++ { - ret = append(ret, logs[i]...) - } - } - return ret -} - // reorg takes two blocks, an old chain and a new chain and will reconstruct the // blocks and inserts them to be part of the new canonical chain and accumulates // potential missing transactions and post an event about them. @@ -1967,9 +2018,6 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error { deletedTxs []common.Hash addedTxs []common.Hash - - deletedLogs [][]*types.Log - rebirthLogs [][]*types.Log ) // Reduce the longer chain to the same number as the shorter one if oldBlock.NumberU64() > newBlock.NumberU64() { @@ -1979,12 +2027,6 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error { for _, tx := range oldBlock.Transactions() { deletedTxs = append(deletedTxs, tx.Hash()) } - - // Collect deleted logs for notification - logs := bc.collectLogs(oldBlock.Hash(), true) - if len(logs) > 0 { - deletedLogs = append(deletedLogs, logs) - } } } else { // New chain is longer, stash all blocks away for subsequent insertion @@ -2011,12 +2053,6 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error { for _, tx := range oldBlock.Transactions() { deletedTxs = append(deletedTxs, tx.Hash()) } - - // Collect deleted logs for notification - logs := bc.collectLogs(oldBlock.Hash(), true) - if len(logs) > 0 { - deletedLogs = append(deletedLogs, logs) - } newChain = append(newChain, newBlock) // Step back with both chains @@ -2090,28 +2126,42 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error { log.Crit("Failed to delete useless indexes", "err", err) } - // Collect the logs - for i := len(newChain) - 1; i >= 1; i-- { - // Collect reborn logs due to chain reorg - logs := bc.collectLogs(newChain[i].Hash(), false) - if len(logs) > 0 { - rebirthLogs = append(rebirthLogs, logs) + // Send out events for logs from the old canon chain, and 'reborn' + // logs from the new canon chain. The number of logs can be very + // high, so the events are sent in batches of size around 512. + + // Deleted logs + blocks: + var deletedLogs []*types.Log + for i := len(oldChain) - 1; i >= 0; i-- { + // Also send event for blocks removed from the canon chain. + bc.chainSideFeed.Send(ChainSideEvent{Block: oldChain[i]}) + + // Collect deleted logs for notification + if logs := bc.collectLogs(oldChain[i].Hash(), true); len(logs) > 0 { + deletedLogs = append(deletedLogs, logs...) + } + if len(deletedLogs) > 512 { + bc.rmLogsFeed.Send(RemovedLogsEvent{deletedLogs}) + deletedLogs = nil } } - // If any logs need to be fired, do it now. In theory we could avoid creating - // this goroutine if there are no events to fire, but realistcally that only - // ever happens if we're reorging empty blocks, which will only happen on idle - // networks where performance is not an issue either way. if len(deletedLogs) > 0 { - bc.rmLogsFeed.Send(RemovedLogsEvent{mergeLogs(deletedLogs, true)}) - } - if len(rebirthLogs) > 0 { - bc.logsFeed.Send(mergeLogs(rebirthLogs, false)) + bc.rmLogsFeed.Send(RemovedLogsEvent{deletedLogs}) } - if len(oldChain) > 0 { - for i := len(oldChain) - 1; i >= 0; i-- { - bc.chainSideFeed.Send(ChainSideEvent{Block: oldChain[i]}) + + // New logs: + var rebirthLogs []*types.Log + for i := len(newChain) - 1; i >= 1; i-- { + if logs := bc.collectLogs(newChain[i].Hash(), false); len(logs) > 0 { + rebirthLogs = append(rebirthLogs, logs...) } + if len(rebirthLogs) > 512 { + bc.logsFeed.Send(rebirthLogs) + rebirthLogs = nil + } + } + if len(rebirthLogs) > 0 { + bc.logsFeed.Send(rebirthLogs) } return nil } @@ -2232,6 +2282,44 @@ func (bc *BlockChain) skipBlock(err error, it *insertIterator) bool { return false } +// indexBlocks reindexes or unindexes transactions depending on user configuration +func (bc *BlockChain) indexBlocks(tail *uint64, head uint64, done chan struct{}) { + defer func() { close(done) }() + + // The tail flag is not existent, it means the node is just initialized + // and all blocks(may from ancient store) are not indexed yet. + if tail == nil { + from := uint64(0) + if bc.txLookupLimit != 0 && head >= bc.txLookupLimit { + from = head - bc.txLookupLimit + 1 + } + rawdb.IndexTransactions(bc.db, from, head+1, bc.quit) + return + } + // The tail flag is existent, but the whole chain is required to be indexed. + if bc.txLookupLimit == 0 || head < bc.txLookupLimit { + if *tail > 0 { + // It can happen when chain is rewound to a historical point which + // is even lower than the indexes tail, recap the indexing target + // to new head to avoid reading non-existent block bodies. + end := *tail + if end > head+1 { + end = head + 1 + } + rawdb.IndexTransactions(bc.db, 0, end, bc.quit) + } + return + } + // Update the transaction index to the new chain state + if head-bc.txLookupLimit+1 < *tail { + // Reindex a part of missing indices and rewind index tail to HEAD-limit + rawdb.IndexTransactions(bc.db, head-bc.txLookupLimit+1, *tail, bc.quit) + } else { + // Unindex a part of stale indices and forward index tail to HEAD-limit + rawdb.UnindexTransactions(bc.db, *tail, head-bc.txLookupLimit+1, bc.quit) + } +} + // maintainTxIndex is responsible for the construction and deletion of the // transaction index. // @@ -2239,65 +2327,13 @@ func (bc *BlockChain) skipBlock(err error, it *insertIterator) bool { // which ancient tx indices get deleted. If `txlookuplimit` is 0, it means // all tx indices will be reserved. // -// The user can adjust the txlookuplimit value for each launch after fast -// sync, Geth will automatically construct the missing indices and delete -// the extra indices. -func (bc *BlockChain) maintainTxIndex(ancients uint64) { +// The user can adjust the txlookuplimit value for each launch after sync, +// Geth will automatically construct the missing indices or delete the extra +// indices. +func (bc *BlockChain) maintainTxIndex() { defer bc.wg.Done() - // Before starting the actual maintenance, we need to handle a special case, - // where user might init Geth with an external ancient database. If so, we - // need to reindex all necessary transactions before starting to process any - // pruning requests. - if ancients > 0 { - var from = uint64(0) - if bc.txLookupLimit != 0 && ancients > bc.txLookupLimit { - from = ancients - bc.txLookupLimit - } - rawdb.IndexTransactions(bc.db, from, ancients, bc.quit) - } - - // indexBlocks reindexes or unindexes transactions depending on user configuration - indexBlocks := func(tail *uint64, head uint64, done chan struct{}) { - defer func() { done <- struct{}{} }() - - // If the user just upgraded Geth to a new version which supports transaction - // index pruning, write the new tail and remove anything older. - if tail == nil { - if bc.txLookupLimit == 0 || head < bc.txLookupLimit { - // Nothing to delete, write the tail and return - rawdb.WriteTxIndexTail(bc.db, 0) - } else { - // Prune all stale tx indices and record the tx index tail - rawdb.UnindexTransactions(bc.db, 0, head-bc.txLookupLimit+1, bc.quit) - } - return - } - // If a previous indexing existed, make sure that we fill in any missing entries - if bc.txLookupLimit == 0 || head < bc.txLookupLimit { - if *tail > 0 { - // It can happen when chain is rewound to a historical point which - // is even lower than the indexes tail, recap the indexing target - // to new head to avoid reading non-existent block bodies. - end := *tail - if end > head+1 { - end = head + 1 - } - rawdb.IndexTransactions(bc.db, 0, end, bc.quit) - } - return - } - // Update the transaction index to the new chain state - if head-bc.txLookupLimit+1 < *tail { - // Reindex a part of missing indices and rewind index tail to HEAD-limit - rawdb.IndexTransactions(bc.db, head-bc.txLookupLimit+1, *tail, bc.quit) - } else { - // Unindex a part of stale indices and forward index tail to HEAD-limit - rawdb.UnindexTransactions(bc.db, *tail, head-bc.txLookupLimit+1, bc.quit) - } - } - - // Any reindexing done, start listening to chain events and moving the index window + // Listening to chain events and manipulate the transaction indexes. var ( done chan struct{} // Non-nil if background unindexing or reindexing routine is active. headCh = make(chan ChainHeadEvent, 1) // Buffered to avoid locking up the event feed @@ -2313,7 +2349,7 @@ func (bc *BlockChain) maintainTxIndex(ancients uint64) { case head := <-headCh: if done == nil { done = make(chan struct{}) - go indexBlocks(rawdb.ReadTxIndexTail(bc.db), head.Block.NumberU64(), done) + go bc.indexBlocks(rawdb.ReadTxIndexTail(bc.db), head.Block.NumberU64(), done) } case <-done: done = nil @@ -2330,24 +2366,32 @@ func (bc *BlockChain) maintainTxIndex(ancients uint64) { // reportBlock logs a bad block error. func (bc *BlockChain) reportBlock(block *types.Block, receipts types.Receipts, err error) { rawdb.WriteBadBlock(bc.db, block) + log.Error(summarizeBadBlock(block, receipts, bc.Config(), err)) +} +// summarizeBadBlock returns a string summarizing the bad block and other +// relevant information. +func summarizeBadBlock(block *types.Block, receipts []*types.Receipt, config *params.ChainConfig, err error) string { var receiptString string for i, receipt := range receipts { - receiptString += fmt.Sprintf("\t %d: cumulative: %v gas: %v contract: %v status: %v tx: %v logs: %v bloom: %x state: %x\n", + receiptString += fmt.Sprintf("\n %d: cumulative: %v gas: %v contract: %v status: %v tx: %v logs: %v bloom: %x state: %x", i, receipt.CumulativeGasUsed, receipt.GasUsed, receipt.ContractAddress.Hex(), receipt.Status, receipt.TxHash.Hex(), receipt.Logs, receipt.Bloom, receipt.PostState) } - log.Error(fmt.Sprintf(` + version, vcs := version.Info() + platform := fmt.Sprintf("%s %s %s %s", version, runtime.Version(), runtime.GOARCH, runtime.GOOS) + if vcs != "" { + vcs = fmt.Sprintf("\nVCS: %s", vcs) + } + return fmt.Sprintf(` ########## BAD BLOCK ######### -Chain config: %v - -Number: %v -Hash: 0x%x -%v - +Block: %v (%#x) Error: %v +Platform: %v%v +Chain config: %#v +Receipts: %v ############################## -`, bc.chainConfig, block.Number(), block.Hash(), receiptString, err)) +`, block.Number(), block.Hash(), err, platform, vcs, config, receiptString) } // InsertHeaderChain attempts to insert the given header chain in to the local @@ -2374,3 +2418,11 @@ func (bc *BlockChain) InsertHeaderChain(chain []*types.Header, checkFreq int) (i _, err := bc.hc.InsertHeaderChain(chain, start, bc.forker) return 0, err } + +// SetBlockValidatorAndProcessorForTesting sets the current validator and processor. +// This method can be used to force an invalid blockchain to be verified for tests. +// This method is unsafe and should only be used before block import starts. +func (bc *BlockChain) SetBlockValidatorAndProcessorForTesting(v Validator, p Processor) { + bc.validator = v + bc.processor = p +} diff --git a/core/blockchain_insert.go b/core/blockchain_insert.go index 479eccc83e47a..8f496e182c9e4 100644 --- a/core/blockchain_insert.go +++ b/core/blockchain_insert.go @@ -56,9 +56,9 @@ func (st *insertStats) report(chain []*types.Block, index int, dirty common.Stor // Assemble the log context and send it to the logger context := []interface{}{ + "number", end.Number(), "hash", end.Hash(), "blocks", st.processed, "txs", txs, "mgas", float64(st.usedGas) / 1000000, "elapsed", common.PrettyDuration(elapsed), "mgasps", float64(st.usedGas) * 1000 / float64(elapsed), - "number", end.Number(), "hash", end.Hash(), } if timestamp := time.Unix(int64(end.Time()), 0); time.Since(timestamp) > time.Minute { context = append(context, []interface{}{"age", common.PrettyAge(timestamp)}...) diff --git a/core/blockchain_reader.go b/core/blockchain_reader.go index b8d4233c6ecd8..5814c8a0daee3 100644 --- a/core/blockchain_reader.go +++ b/core/blockchain_reader.go @@ -55,6 +55,12 @@ func (bc *BlockChain) CurrentFinalizedBlock() *types.Block { return bc.currentFinalizedBlock.Load().(*types.Block) } +// CurrentSafeBlock retrieves the current safe block of the canonical +// chain. The block is retrieved from the blockchain's internal cache. +func (bc *BlockChain) CurrentSafeBlock() *types.Block { + return bc.currentSafeBlock.Load().(*types.Block) +} + // HasHeader checks if a block header is present in the database or not, caching // it if present. func (bc *BlockChain) HasHeader(hash common.Hash, number uint64) bool { @@ -131,6 +137,9 @@ func (bc *BlockChain) HasBlock(hash common.Hash, number uint64) bool { if bc.blockCache.Contains(hash) { return true } + if !bc.HasHeader(hash, number) { + return false + } return rawdb.HasBody(bc.db, hash, number) } diff --git a/core/blockchain_repair_test.go b/core/blockchain_repair_test.go index 24309405d2b3a..6e61f89c3b14e 100644 --- a/core/blockchain_repair_test.go +++ b/core/blockchain_repair_test.go @@ -564,7 +564,7 @@ func testShortReorgedSnapSyncingRepair(t *testing.T, snapshots bool) { // Tests a recovery for a long canonical chain with frozen blocks where a recent // block - newer than the ancient limit - was already committed to disk and then // the process crashed. In this case we expect the chain to be rolled back to the -// committed block, with everything afterwads kept as fast sync data. +// committed block, with everything afterwards kept as fast sync data. func TestLongShallowRepair(t *testing.T) { testLongShallowRepair(t, false) } func TestLongShallowRepairWithSnapshots(t *testing.T) { testLongShallowRepair(t, true) } @@ -609,7 +609,7 @@ func testLongShallowRepair(t *testing.T, snapshots bool) { // Tests a recovery for a long canonical chain with frozen blocks where a recent // block - older than the ancient limit - was already committed to disk and then // the process crashed. In this case we expect the chain to be rolled back to the -// committed block, with everything afterwads deleted. +// committed block, with everything afterwards deleted. func TestLongDeepRepair(t *testing.T) { testLongDeepRepair(t, false) } func TestLongDeepRepairWithSnapshots(t *testing.T) { testLongDeepRepair(t, true) } @@ -653,7 +653,7 @@ func testLongDeepRepair(t *testing.T, snapshots bool) { // Tests a recovery for a long canonical chain with frozen blocks where the fast // sync pivot point - newer than the ancient limit - was already committed, after // which the process crashed. In this case we expect the chain to be rolled back -// to the committed block, with everything afterwads kept as fast sync data. +// to the committed block, with everything afterwards kept as fast sync data. func TestLongSnapSyncedShallowRepair(t *testing.T) { testLongSnapSyncedShallowRepair(t, false) } @@ -702,7 +702,7 @@ func testLongSnapSyncedShallowRepair(t *testing.T, snapshots bool) { // Tests a recovery for a long canonical chain with frozen blocks where the fast // sync pivot point - older than the ancient limit - was already committed, after // which the process crashed. In this case we expect the chain to be rolled back -// to the committed block, with everything afterwads deleted. +// to the committed block, with everything afterwards deleted. func TestLongSnapSyncedDeepRepair(t *testing.T) { testLongSnapSyncedDeepRepair(t, false) } func TestLongSnapSyncedDeepRepairWithSnapshots(t *testing.T) { testLongSnapSyncedDeepRepair(t, true) } @@ -843,7 +843,7 @@ func testLongSnapSyncingDeepRepair(t *testing.T, snapshots bool) { // side chain, where a recent block - newer than the ancient limit - was already // committed to disk and then the process crashed. In this test scenario the side // chain is below the committed block. In this case we expect the chain to be -// rolled back to the committed block, with everything afterwads kept as fast +// rolled back to the committed block, with everything afterwards kept as fast // sync data; the side chain completely nuked by the freezer. func TestLongOldForkedShallowRepair(t *testing.T) { testLongOldForkedShallowRepair(t, false) @@ -895,7 +895,7 @@ func testLongOldForkedShallowRepair(t *testing.T, snapshots bool) { // side chain, where a recent block - older than the ancient limit - was already // committed to disk and then the process crashed. In this test scenario the side // chain is below the committed block. In this case we expect the canonical chain -// to be rolled back to the committed block, with everything afterwads deleted; +// to be rolled back to the committed block, with everything afterwards deleted; // the side chain completely nuked by the freezer. func TestLongOldForkedDeepRepair(t *testing.T) { testLongOldForkedDeepRepair(t, false) } func TestLongOldForkedDeepRepairWithSnapshots(t *testing.T) { testLongOldForkedDeepRepair(t, true) } @@ -942,7 +942,7 @@ func testLongOldForkedDeepRepair(t *testing.T, snapshots bool) { // side chain, where the fast sync pivot point - newer than the ancient limit - // was already committed to disk and then the process crashed. In this test scenario // the side chain is below the committed block. In this case we expect the chain -// to be rolled back to the committed block, with everything afterwads kept as +// to be rolled back to the committed block, with everything afterwards kept as // fast sync data; the side chain completely nuked by the freezer. func TestLongOldForkedSnapSyncedShallowRepair(t *testing.T) { testLongOldForkedSnapSyncedShallowRepair(t, false) @@ -994,7 +994,7 @@ func testLongOldForkedSnapSyncedShallowRepair(t *testing.T, snapshots bool) { // side chain, where the fast sync pivot point - older than the ancient limit - // was already committed to disk and then the process crashed. In this test scenario // the side chain is below the committed block. In this case we expect the canonical -// chain to be rolled back to the committed block, with everything afterwads deleted; +// chain to be rolled back to the committed block, with everything afterwards deleted; // the side chain completely nuked by the freezer. func TestLongOldForkedSnapSyncedDeepRepair(t *testing.T) { testLongOldForkedSnapSyncedDeepRepair(t, false) @@ -1149,7 +1149,7 @@ func testLongOldForkedSnapSyncingDeepRepair(t *testing.T, snapshots bool) { // side chain, where a recent block - newer than the ancient limit - was already // committed to disk and then the process crashed. In this test scenario the side // chain is above the committed block. In this case we expect the chain to be -// rolled back to the committed block, with everything afterwads kept as fast +// rolled back to the committed block, with everything afterwards kept as fast // sync data; the side chain completely nuked by the freezer. func TestLongNewerForkedShallowRepair(t *testing.T) { testLongNewerForkedShallowRepair(t, false) @@ -1201,7 +1201,7 @@ func testLongNewerForkedShallowRepair(t *testing.T, snapshots bool) { // side chain, where a recent block - older than the ancient limit - was already // committed to disk and then the process crashed. In this test scenario the side // chain is above the committed block. In this case we expect the canonical chain -// to be rolled back to the committed block, with everything afterwads deleted; +// to be rolled back to the committed block, with everything afterwards deleted; // the side chain completely nuked by the freezer. func TestLongNewerForkedDeepRepair(t *testing.T) { testLongNewerForkedDeepRepair(t, false) } func TestLongNewerForkedDeepRepairWithSnapshots(t *testing.T) { testLongNewerForkedDeepRepair(t, true) } @@ -1248,7 +1248,7 @@ func testLongNewerForkedDeepRepair(t *testing.T, snapshots bool) { // side chain, where the fast sync pivot point - newer than the ancient limit - // was already committed to disk and then the process crashed. In this test scenario // the side chain is above the committed block. In this case we expect the chain -// to be rolled back to the committed block, with everything afterwads kept as fast +// to be rolled back to the committed block, with everything afterwards kept as fast // sync data; the side chain completely nuked by the freezer. func TestLongNewerForkedSnapSyncedShallowRepair(t *testing.T) { testLongNewerForkedSnapSyncedShallowRepair(t, false) @@ -1300,7 +1300,7 @@ func testLongNewerForkedSnapSyncedShallowRepair(t *testing.T, snapshots bool) { // side chain, where the fast sync pivot point - older than the ancient limit - // was already committed to disk and then the process crashed. In this test scenario // the side chain is above the committed block. In this case we expect the canonical -// chain to be rolled back to the committed block, with everything afterwads deleted; +// chain to be rolled back to the committed block, with everything afterwards deleted; // the side chain completely nuked by the freezer. func TestLongNewerForkedSnapSyncedDeepRepair(t *testing.T) { testLongNewerForkedSnapSyncedDeepRepair(t, false) @@ -1454,7 +1454,7 @@ func testLongNewerForkedSnapSyncingDeepRepair(t *testing.T, snapshots bool) { // Tests a recovery for a long canonical chain with frozen blocks and a longer side // chain, where a recent block - newer than the ancient limit - was already committed // to disk and then the process crashed. In this case we expect the chain to be -// rolled back to the committed block, with everything afterwads kept as fast sync +// rolled back to the committed block, with everything afterwards kept as fast sync // data. The side chain completely nuked by the freezer. func TestLongReorgedShallowRepair(t *testing.T) { testLongReorgedShallowRepair(t, false) } func TestLongReorgedShallowRepairWithSnapshots(t *testing.T) { testLongReorgedShallowRepair(t, true) } @@ -1501,7 +1501,7 @@ func testLongReorgedShallowRepair(t *testing.T, snapshots bool) { // Tests a recovery for a long canonical chain with frozen blocks and a longer side // chain, where a recent block - older than the ancient limit - was already committed // to disk and then the process crashed. In this case we expect the canonical chains -// to be rolled back to the committed block, with everything afterwads deleted. The +// to be rolled back to the committed block, with everything afterwards deleted. The // side chain completely nuked by the freezer. func TestLongReorgedDeepRepair(t *testing.T) { testLongReorgedDeepRepair(t, false) } func TestLongReorgedDeepRepairWithSnapshots(t *testing.T) { testLongReorgedDeepRepair(t, true) } @@ -1548,7 +1548,7 @@ func testLongReorgedDeepRepair(t *testing.T, snapshots bool) { // side chain, where the fast sync pivot point - newer than the ancient limit - // was already committed to disk and then the process crashed. In this case we // expect the chain to be rolled back to the committed block, with everything -// afterwads kept as fast sync data. The side chain completely nuked by the +// afterwards kept as fast sync data. The side chain completely nuked by the // freezer. func TestLongReorgedSnapSyncedShallowRepair(t *testing.T) { testLongReorgedSnapSyncedShallowRepair(t, false) @@ -1600,7 +1600,7 @@ func testLongReorgedSnapSyncedShallowRepair(t *testing.T, snapshots bool) { // side chain, where the fast sync pivot point - older than the ancient limit - // was already committed to disk and then the process crashed. In this case we // expect the canonical chains to be rolled back to the committed block, with -// everything afterwads deleted. The side chain completely nuked by the freezer. +// everything afterwards deleted. The side chain completely nuked by the freezer. func TestLongReorgedSnapSyncedDeepRepair(t *testing.T) { testLongReorgedSnapSyncedDeepRepair(t, false) } @@ -1764,9 +1764,12 @@ func testRepair(t *testing.T, tt *rewindTest, snapshots bool) { // Initialize a fresh chain var ( - genesis = (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(db) - engine = ethash.NewFullFaker() - config = &CacheConfig{ + gspec = &Genesis{ + BaseFee: big.NewInt(params.InitialBaseFee), + Config: params.AllEthashProtocolChanges, + } + engine = ethash.NewFullFaker() + config = &CacheConfig{ TrieCleanLimit: 256, TrieDirtyLimit: 256, TrieTimeLimit: 5 * time.Minute, @@ -1778,21 +1781,21 @@ func testRepair(t *testing.T, tt *rewindTest, snapshots bool) { config.SnapshotLimit = 256 config.SnapshotWait = true } - chain, err := NewBlockChain(db, config, params.AllEthashProtocolChanges, engine, vm.Config{}, nil, nil) + chain, err := NewBlockChain(db, config, gspec, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to create chain: %v", err) } // If sidechain blocks are needed, make a light chain and import it var sideblocks types.Blocks if tt.sidechainBlocks > 0 { - sideblocks, _ = GenerateChain(params.TestChainConfig, genesis, engine, rawdb.NewMemoryDatabase(), tt.sidechainBlocks, func(i int, b *BlockGen) { + sideblocks, _ = GenerateChain(gspec.Config, gspec.ToBlock(), engine, rawdb.NewMemoryDatabase(), tt.sidechainBlocks, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{0x01}) }) if _, err := chain.InsertChain(sideblocks); err != nil { t.Fatalf("Failed to import side chain: %v", err) } } - canonblocks, _ := GenerateChain(params.TestChainConfig, genesis, engine, rawdb.NewMemoryDatabase(), tt.canonicalBlocks, func(i int, b *BlockGen) { + canonblocks, _ := GenerateChain(gspec.Config, gspec.ToBlock(), engine, rawdb.NewMemoryDatabase(), tt.canonicalBlocks, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{0x02}) b.SetDifficulty(big.NewInt(1000000)) }) @@ -1831,7 +1834,7 @@ func testRepair(t *testing.T, tt *rewindTest, snapshots bool) { } defer db.Close() - newChain, err := NewBlockChain(db, nil, params.AllEthashProtocolChanges, engine, vm.Config{}, nil, nil) + newChain, err := NewBlockChain(db, nil, gspec, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to recreate chain: %v", err) } @@ -1888,9 +1891,12 @@ func TestIssue23496(t *testing.T) { // Initialize a fresh chain var ( - genesis = (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(db) - engine = ethash.NewFullFaker() - config = &CacheConfig{ + gspec = &Genesis{ + Config: params.TestChainConfig, + BaseFee: big.NewInt(params.InitialBaseFee), + } + engine = ethash.NewFullFaker() + config = &CacheConfig{ TrieCleanLimit: 256, TrieDirtyLimit: 256, TrieTimeLimit: 5 * time.Minute, @@ -1898,11 +1904,11 @@ func TestIssue23496(t *testing.T) { SnapshotWait: true, } ) - chain, err := NewBlockChain(db, config, params.AllEthashProtocolChanges, engine, vm.Config{}, nil, nil) + chain, err := NewBlockChain(db, config, gspec, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to create chain: %v", err) } - blocks, _ := GenerateChain(params.TestChainConfig, genesis, engine, rawdb.NewMemoryDatabase(), 4, func(i int, b *BlockGen) { + _, blocks, _ := GenerateChainWithGenesis(gspec, engine, 4, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{0x02}) b.SetDifficulty(big.NewInt(1000000)) }) @@ -1942,7 +1948,7 @@ func TestIssue23496(t *testing.T) { } defer db.Close() - chain, err = NewBlockChain(db, nil, params.AllEthashProtocolChanges, engine, vm.Config{}, nil, nil) + chain, err = NewBlockChain(db, nil, gspec, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to recreate chain: %v", err) } diff --git a/core/blockchain_sethead_test.go b/core/blockchain_sethead_test.go index 970e0306308dd..1eb588d02fd1e 100644 --- a/core/blockchain_sethead_test.go +++ b/core/blockchain_sethead_test.go @@ -51,6 +51,7 @@ type rewindTest struct { expHeadBlock uint64 // Block number of the expected head full block } +//nolint:unused func (tt *rewindTest) dump(crash bool) string { buffer := new(strings.Builder) @@ -1963,9 +1964,12 @@ func testSetHead(t *testing.T, tt *rewindTest, snapshots bool) { // Initialize a fresh chain var ( - genesis = (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(db) - engine = ethash.NewFullFaker() - config = &CacheConfig{ + gspec = &Genesis{ + BaseFee: big.NewInt(params.InitialBaseFee), + Config: params.AllEthashProtocolChanges, + } + engine = ethash.NewFullFaker() + config = &CacheConfig{ TrieCleanLimit: 256, TrieDirtyLimit: 256, TrieTimeLimit: 5 * time.Minute, @@ -1976,21 +1980,21 @@ func testSetHead(t *testing.T, tt *rewindTest, snapshots bool) { config.SnapshotLimit = 256 config.SnapshotWait = true } - chain, err := NewBlockChain(db, config, params.AllEthashProtocolChanges, engine, vm.Config{}, nil, nil) + chain, err := NewBlockChain(db, config, gspec, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to create chain: %v", err) } // If sidechain blocks are needed, make a light chain and import it var sideblocks types.Blocks if tt.sidechainBlocks > 0 { - sideblocks, _ = GenerateChain(params.TestChainConfig, genesis, engine, rawdb.NewMemoryDatabase(), tt.sidechainBlocks, func(i int, b *BlockGen) { + sideblocks, _ = GenerateChain(gspec.Config, gspec.ToBlock(), engine, rawdb.NewMemoryDatabase(), tt.sidechainBlocks, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{0x01}) }) if _, err := chain.InsertChain(sideblocks); err != nil { t.Fatalf("Failed to import side chain: %v", err) } } - canonblocks, _ := GenerateChain(params.TestChainConfig, genesis, engine, rawdb.NewMemoryDatabase(), tt.canonicalBlocks, func(i int, b *BlockGen) { + canonblocks, _ := GenerateChain(gspec.Config, gspec.ToBlock(), engine, rawdb.NewMemoryDatabase(), tt.canonicalBlocks, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{0x02}) b.SetDifficulty(big.NewInt(1000000)) }) diff --git a/core/blockchain_snapshot_test.go b/core/blockchain_snapshot_test.go index dfa8ed65ec6d5..1b38ad51e985f 100644 --- a/core/blockchain_snapshot_test.go +++ b/core/blockchain_snapshot_test.go @@ -52,8 +52,9 @@ type snapshotTestBasic struct { // share fields, set in runtime datadir string db ethdb.Database - gendb ethdb.Database + genDb ethdb.Database engine consensus.Engine + gspec *Genesis } func (basic *snapshotTestBasic) prepare(t *testing.T) (*BlockChain, []*types.Block) { @@ -66,20 +67,22 @@ func (basic *snapshotTestBasic) prepare(t *testing.T) (*BlockChain, []*types.Blo } // Initialize a fresh chain var ( - genesis = (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(db) - engine = ethash.NewFullFaker() - gendb = rawdb.NewMemoryDatabase() + gspec = &Genesis{ + BaseFee: big.NewInt(params.InitialBaseFee), + Config: params.AllEthashProtocolChanges, + } + engine = ethash.NewFullFaker() // Snapshot is enabled, the first snapshot is created from the Genesis. // The snapshot memory allowance is 256MB, it means no snapshot flush // will happen during the block insertion. cacheConfig = defaultCacheConfig ) - chain, err := NewBlockChain(db, cacheConfig, params.AllEthashProtocolChanges, engine, vm.Config{}, nil, nil) + chain, err := NewBlockChain(db, cacheConfig, gspec, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to create chain: %v", err) } - blocks, _ := GenerateChain(params.TestChainConfig, genesis, engine, gendb, basic.chainBlocks, func(i int, b *BlockGen) {}) + genDb, blocks, _ := GenerateChainWithGenesis(gspec, engine, basic.chainBlocks, func(i int, b *BlockGen) {}) // Insert the blocks with configured settings. var breakpoints []uint64 @@ -116,8 +119,9 @@ func (basic *snapshotTestBasic) prepare(t *testing.T) (*BlockChain, []*types.Blo // Set runtime fields basic.datadir = datadir basic.db = db - basic.gendb = gendb + basic.genDb = genDb basic.engine = engine + basic.gspec = gspec return chain, blocks } @@ -150,6 +154,7 @@ func (basic *snapshotTestBasic) verify(t *testing.T, chain *BlockChain, blocks [ } } +//nolint:unused func (basic *snapshotTestBasic) dump() string { buffer := new(strings.Builder) @@ -200,7 +205,7 @@ func (basic *snapshotTestBasic) dump() string { func (basic *snapshotTestBasic) teardown() { basic.db.Close() - basic.gendb.Close() + basic.genDb.Close() os.RemoveAll(basic.datadir) } @@ -218,7 +223,7 @@ func (snaptest *snapshotTest) test(t *testing.T) { // Restart the chain normally chain.Stop() - newchain, err := NewBlockChain(snaptest.db, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil) + newchain, err := NewBlockChain(snaptest.db, nil, snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to recreate chain: %v", err) } @@ -254,13 +259,13 @@ func (snaptest *crashSnapshotTest) test(t *testing.T) { // the crash, we do restart twice here: one after the crash and one // after the normal stop. It's used to ensure the broken snapshot // can be detected all the time. - newchain, err := NewBlockChain(newdb, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil) + newchain, err := NewBlockChain(newdb, nil, snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to recreate chain: %v", err) } newchain.Stop() - newchain, err = NewBlockChain(newdb, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil) + newchain, err = NewBlockChain(newdb, nil, snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to recreate chain: %v", err) } @@ -287,7 +292,7 @@ func (snaptest *gappedSnapshotTest) test(t *testing.T) { // Insert blocks without enabling snapshot if gapping is required. chain.Stop() - gappedBlocks, _ := GenerateChain(params.TestChainConfig, blocks[len(blocks)-1], snaptest.engine, snaptest.gendb, snaptest.gapped, func(i int, b *BlockGen) {}) + gappedBlocks, _ := GenerateChain(params.TestChainConfig, blocks[len(blocks)-1], snaptest.engine, snaptest.genDb, snaptest.gapped, func(i int, b *BlockGen) {}) // Insert a few more blocks without enabling snapshot var cacheConfig = &CacheConfig{ @@ -296,7 +301,7 @@ func (snaptest *gappedSnapshotTest) test(t *testing.T) { TrieTimeLimit: 5 * time.Minute, SnapshotLimit: 0, } - newchain, err := NewBlockChain(snaptest.db, cacheConfig, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil) + newchain, err := NewBlockChain(snaptest.db, cacheConfig, snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to recreate chain: %v", err) } @@ -304,7 +309,7 @@ func (snaptest *gappedSnapshotTest) test(t *testing.T) { newchain.Stop() // Restart the chain with enabling the snapshot - newchain, err = NewBlockChain(snaptest.db, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil) + newchain, err = NewBlockChain(snaptest.db, nil, snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to recreate chain: %v", err) } @@ -332,55 +337,7 @@ func (snaptest *setHeadSnapshotTest) test(t *testing.T) { chain.SetHead(snaptest.setHead) chain.Stop() - newchain, err := NewBlockChain(snaptest.db, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil) - if err != nil { - t.Fatalf("Failed to recreate chain: %v", err) - } - defer newchain.Stop() - - snaptest.verify(t, newchain, blocks) -} - -// restartCrashSnapshotTest is the test type used to test this scenario: -// - have a complete snapshot -// - restart chain -// - insert more blocks with enabling the snapshot -// - commit the snapshot -// - crash -// - restart again -type restartCrashSnapshotTest struct { - snapshotTestBasic - newBlocks int -} - -func (snaptest *restartCrashSnapshotTest) test(t *testing.T) { - // It's hard to follow the test case, visualize the input - // log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true)))) - // fmt.Println(tt.dump()) - chain, blocks := snaptest.prepare(t) - - // Firstly, stop the chain properly, with all snapshot journal - // and state committed. - chain.Stop() - - newchain, err := NewBlockChain(snaptest.db, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil) - if err != nil { - t.Fatalf("Failed to recreate chain: %v", err) - } - newBlocks, _ := GenerateChain(params.TestChainConfig, blocks[len(blocks)-1], snaptest.engine, snaptest.gendb, snaptest.newBlocks, func(i int, b *BlockGen) {}) - newchain.InsertChain(newBlocks) - - // Commit the entire snapshot into the disk if requested. Note only - // (a) snapshot root and (b) snapshot generator will be committed, - // the diff journal is not. - newchain.Snapshots().Cap(newBlocks[len(newBlocks)-1].Root(), 0) - - // Simulate the blockchain crash - // Don't call chain.Stop here, so that no snapshot - // journal and latest state will be committed - - // Restart the chain after the crash - newchain, err = NewBlockChain(snaptest.db, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil) + newchain, err := NewBlockChain(snaptest.db, nil, snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to recreate chain: %v", err) } @@ -415,11 +372,11 @@ func (snaptest *wipeCrashSnapshotTest) test(t *testing.T) { TrieTimeLimit: 5 * time.Minute, SnapshotLimit: 0, } - newchain, err := NewBlockChain(snaptest.db, config, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil) + newchain, err := NewBlockChain(snaptest.db, config, snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to recreate chain: %v", err) } - newBlocks, _ := GenerateChain(params.TestChainConfig, blocks[len(blocks)-1], snaptest.engine, snaptest.gendb, snaptest.newBlocks, func(i int, b *BlockGen) {}) + newBlocks, _ := GenerateChain(params.TestChainConfig, blocks[len(blocks)-1], snaptest.engine, snaptest.genDb, snaptest.newBlocks, func(i int, b *BlockGen) {}) newchain.InsertChain(newBlocks) newchain.Stop() @@ -431,13 +388,12 @@ func (snaptest *wipeCrashSnapshotTest) test(t *testing.T) { SnapshotLimit: 256, SnapshotWait: false, // Don't wait rebuild } - newchain, err = NewBlockChain(snaptest.db, config, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil) + _, err = NewBlockChain(snaptest.db, config, snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to recreate chain: %v", err) } // Simulate the blockchain crash. - - newchain, err = NewBlockChain(snaptest.db, nil, params.AllEthashProtocolChanges, snaptest.engine, vm.Config{}, nil, nil) + newchain, err = NewBlockChain(snaptest.db, nil, snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to recreate chain: %v", err) } diff --git a/core/blockchain_test.go b/core/blockchain_test.go index 28d3ccd9269b2..aba50c8627c96 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -50,29 +50,32 @@ var ( // newCanonical creates a chain database, and injects a deterministic canonical // chain. Depending on the full flag, if creates either a full block chain or a -// header only chain. -func newCanonical(engine consensus.Engine, n int, full bool) (ethdb.Database, *BlockChain, error) { +// header only chain. The database and genesis specification for block generation +// are also returned in case more test blocks are needed later. +func newCanonical(engine consensus.Engine, n int, full bool) (ethdb.Database, *Genesis, *BlockChain, error) { var ( - db = rawdb.NewMemoryDatabase() - genesis = (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(db) + genesis = &Genesis{ + BaseFee: big.NewInt(params.InitialBaseFee), + Config: params.AllEthashProtocolChanges, + } ) - // Initialize a fresh chain with only a genesis block - blockchain, _ := NewBlockChain(db, nil, params.AllEthashProtocolChanges, engine, vm.Config{}, nil, nil) + blockchain, _ := NewBlockChain(rawdb.NewMemoryDatabase(), nil, genesis, nil, engine, vm.Config{}, nil, nil) + // Create and inject the requested chain if n == 0 { - return db, blockchain, nil + return rawdb.NewMemoryDatabase(), genesis, blockchain, nil } if full { // Full block-chain requested - blocks := makeBlockChain(genesis, n, engine, db, canonicalSeed) + genDb, blocks := makeBlockChainWithGenesis(genesis, n, engine, canonicalSeed) _, err := blockchain.InsertChain(blocks) - return db, blockchain, err + return genDb, genesis, blockchain, err } // Header-only chain requested - headers := makeHeaderChain(genesis.Header(), n, engine, db, canonicalSeed) + genDb, headers := makeHeaderChainWithGenesis(genesis, n, engine, canonicalSeed) _, err := blockchain.InsertHeaderChain(headers, 1) - return db, blockchain, err + return genDb, genesis, blockchain, err } func newGwei(n int64) *big.Int { @@ -82,7 +85,7 @@ func newGwei(n int64) *big.Int { // Test fork of length N starting from block i func testFork(t *testing.T, blockchain *BlockChain, i, n int, full bool, comparator func(td1, td2 *big.Int)) { // Copy old chain up to #i into a new db - db, blockchain2, err := newCanonical(ethash.NewFaker(), i, full) + genDb, _, blockchain2, err := newCanonical(ethash.NewFaker(), i, full) if err != nil { t.Fatal("could not make new canonical in testFork", err) } @@ -106,12 +109,12 @@ func testFork(t *testing.T, blockchain *BlockChain, i, n int, full bool, compara headerChainB []*types.Header ) if full { - blockChainB = makeBlockChain(blockchain2.CurrentBlock(), n, ethash.NewFaker(), db, forkSeed) + blockChainB = makeBlockChain(blockchain2.chainConfig, blockchain2.CurrentBlock(), n, ethash.NewFaker(), genDb, forkSeed) if _, err := blockchain2.InsertChain(blockChainB); err != nil { t.Fatalf("failed to insert forking chain: %v", err) } } else { - headerChainB = makeHeaderChain(blockchain2.CurrentHeader(), n, ethash.NewFaker(), db, forkSeed) + headerChainB = makeHeaderChain(blockchain2.chainConfig, blockchain2.CurrentHeader(), n, ethash.NewFaker(), genDb, forkSeed) if _, err := blockchain2.InsertHeaderChain(headerChainB, 1); err != nil { t.Fatalf("failed to insert forking chain: %v", err) } @@ -197,13 +200,13 @@ func testHeaderChainImport(chain []*types.Header, blockchain *BlockChain) error } func TestLastBlock(t *testing.T) { - _, blockchain, err := newCanonical(ethash.NewFaker(), 0, true) + genDb, _, blockchain, err := newCanonical(ethash.NewFaker(), 0, true) if err != nil { t.Fatalf("failed to create pristine chain: %v", err) } defer blockchain.Stop() - blocks := makeBlockChain(blockchain.CurrentBlock(), 1, ethash.NewFullFaker(), blockchain.db, 0) + blocks := makeBlockChain(blockchain.chainConfig, blockchain.CurrentBlock(), 1, ethash.NewFullFaker(), genDb, 0) if _, err := blockchain.InsertChain(blocks); err != nil { t.Fatalf("Failed to insert block: %v", err) } @@ -216,7 +219,7 @@ func TestLastBlock(t *testing.T) { // The chain is reorged to whatever specified. func testInsertAfterMerge(t *testing.T, blockchain *BlockChain, i, n int, full bool) { // Copy old chain up to #i into a new db - db, blockchain2, err := newCanonical(ethash.NewFaker(), i, full) + genDb, _, blockchain2, err := newCanonical(ethash.NewFaker(), i, full) if err != nil { t.Fatal("could not make new canonical in testFork", err) } @@ -237,7 +240,7 @@ func testInsertAfterMerge(t *testing.T, blockchain *BlockChain, i, n int, full b // Extend the newly created chain if full { - blockChainB := makeBlockChain(blockchain2.CurrentBlock(), n, ethash.NewFaker(), db, forkSeed) + blockChainB := makeBlockChain(blockchain2.chainConfig, blockchain2.CurrentBlock(), n, ethash.NewFaker(), genDb, forkSeed) if _, err := blockchain2.InsertChain(blockChainB); err != nil { t.Fatalf("failed to insert forking chain: %v", err) } @@ -248,7 +251,7 @@ func testInsertAfterMerge(t *testing.T, blockchain *BlockChain, i, n int, full b t.Fatalf("failed to reorg to the given chain") } } else { - headerChainB := makeHeaderChain(blockchain2.CurrentHeader(), n, ethash.NewFaker(), db, forkSeed) + headerChainB := makeHeaderChain(blockchain2.chainConfig, blockchain2.CurrentHeader(), n, ethash.NewFaker(), genDb, forkSeed) if _, err := blockchain2.InsertHeaderChain(headerChainB, 1); err != nil { t.Fatalf("failed to insert forking chain: %v", err) } @@ -270,7 +273,7 @@ func testExtendCanonical(t *testing.T, full bool) { length := 5 // Make first chain starting from genesis - _, processor, err := newCanonical(ethash.NewFaker(), length, full) + _, _, processor, err := newCanonical(ethash.NewFaker(), length, full) if err != nil { t.Fatalf("failed to make new canonical chain: %v", err) } @@ -298,7 +301,7 @@ func testExtendCanonicalAfterMerge(t *testing.T, full bool) { length := 5 // Make first chain starting from genesis - _, processor, err := newCanonical(ethash.NewFaker(), length, full) + _, _, processor, err := newCanonical(ethash.NewFaker(), length, full) if err != nil { t.Fatalf("failed to make new canonical chain: %v", err) } @@ -317,7 +320,7 @@ func testShorterFork(t *testing.T, full bool) { length := 10 // Make first chain starting from genesis - _, processor, err := newCanonical(ethash.NewFaker(), length, full) + _, _, processor, err := newCanonical(ethash.NewFaker(), length, full) if err != nil { t.Fatalf("failed to make new canonical chain: %v", err) } @@ -347,7 +350,7 @@ func testShorterForkAfterMerge(t *testing.T, full bool) { length := 10 // Make first chain starting from genesis - _, processor, err := newCanonical(ethash.NewFaker(), length, full) + _, _, processor, err := newCanonical(ethash.NewFaker(), length, full) if err != nil { t.Fatalf("failed to make new canonical chain: %v", err) } @@ -370,7 +373,7 @@ func testLongerFork(t *testing.T, full bool) { length := 10 // Make first chain starting from genesis - _, processor, err := newCanonical(ethash.NewFaker(), length, full) + _, _, processor, err := newCanonical(ethash.NewFaker(), length, full) if err != nil { t.Fatalf("failed to make new canonical chain: %v", err) } @@ -393,7 +396,7 @@ func testLongerForkAfterMerge(t *testing.T, full bool) { length := 10 // Make first chain starting from genesis - _, processor, err := newCanonical(ethash.NewFaker(), length, full) + _, _, processor, err := newCanonical(ethash.NewFaker(), length, full) if err != nil { t.Fatalf("failed to make new canonical chain: %v", err) } @@ -416,7 +419,7 @@ func testEqualFork(t *testing.T, full bool) { length := 10 // Make first chain starting from genesis - _, processor, err := newCanonical(ethash.NewFaker(), length, full) + _, _, processor, err := newCanonical(ethash.NewFaker(), length, full) if err != nil { t.Fatalf("failed to make new canonical chain: %v", err) } @@ -446,7 +449,7 @@ func testEqualForkAfterMerge(t *testing.T, full bool) { length := 10 // Make first chain starting from genesis - _, processor, err := newCanonical(ethash.NewFaker(), length, full) + _, _, processor, err := newCanonical(ethash.NewFaker(), length, full) if err != nil { t.Fatalf("failed to make new canonical chain: %v", err) } @@ -466,7 +469,7 @@ func TestBrokenBlockChain(t *testing.T) { testBrokenChain(t, true) } func testBrokenChain(t *testing.T, full bool) { // Make chain starting from genesis - db, blockchain, err := newCanonical(ethash.NewFaker(), 10, full) + genDb, _, blockchain, err := newCanonical(ethash.NewFaker(), 10, full) if err != nil { t.Fatalf("failed to make new canonical chain: %v", err) } @@ -474,12 +477,12 @@ func testBrokenChain(t *testing.T, full bool) { // Create a forked chain, and try to insert with a missing link if full { - chain := makeBlockChain(blockchain.CurrentBlock(), 5, ethash.NewFaker(), db, forkSeed)[1:] + chain := makeBlockChain(blockchain.chainConfig, blockchain.CurrentBlock(), 5, ethash.NewFaker(), genDb, forkSeed)[1:] if err := testBlockChainImport(chain, blockchain); err == nil { t.Errorf("broken block chain not reported") } } else { - chain := makeHeaderChain(blockchain.CurrentHeader(), 5, ethash.NewFaker(), db, forkSeed)[1:] + chain := makeHeaderChain(blockchain.chainConfig, blockchain.CurrentHeader(), 5, ethash.NewFaker(), genDb, forkSeed)[1:] if err := testHeaderChainImport(chain, blockchain); err == nil { t.Errorf("broken header chain not reported") } @@ -503,7 +506,7 @@ func TestReorgShortBlocks(t *testing.T) { testReorgShort(t, true) } func testReorgShort(t *testing.T, full bool) { // Create a long easy chain vs. a short heavy one. Due to difficulty adjustment // we need a fairly long chain of blocks with different difficulties for a short - // one to become heavyer than a long one. The 96 is an empirical value. + // one to become heavier than a long one. The 96 is an empirical value. easy := make([]int64, 96) for i := 0; i < len(easy); i++ { easy[i] = 60 @@ -517,17 +520,17 @@ func testReorgShort(t *testing.T, full bool) { func testReorg(t *testing.T, first, second []int64, td int64, full bool) { // Create a pristine chain and database - db, blockchain, err := newCanonical(ethash.NewFaker(), 0, full) + genDb, _, blockchain, err := newCanonical(ethash.NewFaker(), 0, full) if err != nil { t.Fatalf("failed to create pristine chain: %v", err) } defer blockchain.Stop() // Insert an easy and a difficult chain afterwards - easyBlocks, _ := GenerateChain(params.TestChainConfig, blockchain.CurrentBlock(), ethash.NewFaker(), db, len(first), func(i int, b *BlockGen) { + easyBlocks, _ := GenerateChain(params.TestChainConfig, blockchain.CurrentBlock(), ethash.NewFaker(), genDb, len(first), func(i int, b *BlockGen) { b.OffsetTime(first[i]) }) - diffBlocks, _ := GenerateChain(params.TestChainConfig, blockchain.CurrentBlock(), ethash.NewFaker(), db, len(second), func(i int, b *BlockGen) { + diffBlocks, _ := GenerateChain(params.TestChainConfig, blockchain.CurrentBlock(), ethash.NewFaker(), genDb, len(second), func(i int, b *BlockGen) { b.OffsetTime(second[i]) }) if full { @@ -590,7 +593,7 @@ func TestBadBlockHashes(t *testing.T) { testBadHashes(t, true) } func testBadHashes(t *testing.T, full bool) { // Create a pristine chain and database - db, blockchain, err := newCanonical(ethash.NewFaker(), 0, full) + genDb, _, blockchain, err := newCanonical(ethash.NewFaker(), 0, full) if err != nil { t.Fatalf("failed to create pristine chain: %v", err) } @@ -598,14 +601,14 @@ func testBadHashes(t *testing.T, full bool) { // Create a chain, ban a hash and try to import if full { - blocks := makeBlockChain(blockchain.CurrentBlock(), 3, ethash.NewFaker(), db, 10) + blocks := makeBlockChain(blockchain.chainConfig, blockchain.CurrentBlock(), 3, ethash.NewFaker(), genDb, 10) BadHashes[blocks[2].Header().Hash()] = true defer func() { delete(BadHashes, blocks[2].Header().Hash()) }() _, err = blockchain.InsertChain(blocks) } else { - headers := makeHeaderChain(blockchain.CurrentHeader(), 3, ethash.NewFaker(), db, 10) + headers := makeHeaderChain(blockchain.chainConfig, blockchain.CurrentHeader(), 3, ethash.NewFaker(), genDb, 10) BadHashes[headers[2].Hash()] = true defer func() { delete(BadHashes, headers[2].Hash()) }() @@ -624,13 +627,13 @@ func TestReorgBadBlockHashes(t *testing.T) { testReorgBadHashes(t, true) } func testReorgBadHashes(t *testing.T, full bool) { // Create a pristine chain and database - db, blockchain, err := newCanonical(ethash.NewFaker(), 0, full) + genDb, gspec, blockchain, err := newCanonical(ethash.NewFaker(), 0, full) if err != nil { t.Fatalf("failed to create pristine chain: %v", err) } // Create a chain, import and ban afterwards - headers := makeHeaderChain(blockchain.CurrentHeader(), 4, ethash.NewFaker(), db, 10) - blocks := makeBlockChain(blockchain.CurrentBlock(), 4, ethash.NewFaker(), db, 10) + headers := makeHeaderChain(blockchain.chainConfig, blockchain.CurrentHeader(), 4, ethash.NewFaker(), genDb, 10) + blocks := makeBlockChain(blockchain.chainConfig, blockchain.CurrentBlock(), 4, ethash.NewFaker(), genDb, 10) if full { if _, err = blockchain.InsertChain(blocks); err != nil { @@ -654,7 +657,7 @@ func testReorgBadHashes(t *testing.T, full bool) { blockchain.Stop() // Create a new BlockChain and check that it rolled back the state. - ncm, err := NewBlockChain(blockchain.db, nil, blockchain.chainConfig, ethash.NewFaker(), vm.Config{}, nil, nil) + ncm, err := NewBlockChain(blockchain.db, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) if err != nil { t.Fatalf("failed to create new chain manager: %v", err) } @@ -680,7 +683,7 @@ func TestBlocksInsertNonceError(t *testing.T) { testInsertNonceError(t, true) } func testInsertNonceError(t *testing.T, full bool) { for i := 1; i < 25 && !t.Failed(); i++ { // Create a pristine chain and database - db, blockchain, err := newCanonical(ethash.NewFaker(), 0, full) + genDb, _, blockchain, err := newCanonical(ethash.NewFaker(), 0, full) if err != nil { t.Fatalf("failed to create pristine chain: %v", err) } @@ -693,7 +696,7 @@ func testInsertNonceError(t *testing.T, full bool) { failNum uint64 ) if full { - blocks := makeBlockChain(blockchain.CurrentBlock(), i, ethash.NewFaker(), db, 0) + blocks := makeBlockChain(blockchain.chainConfig, blockchain.CurrentBlock(), i, ethash.NewFaker(), genDb, 0) failAt = rand.Int() % len(blocks) failNum = blocks[failAt].NumberU64() @@ -701,7 +704,7 @@ func testInsertNonceError(t *testing.T, full bool) { blockchain.engine = ethash.NewFakeFailer(failNum) failRes, err = blockchain.InsertChain(blocks) } else { - headers := makeHeaderChain(blockchain.CurrentHeader(), i, ethash.NewFaker(), db, 0) + headers := makeHeaderChain(blockchain.chainConfig, blockchain.CurrentHeader(), i, ethash.NewFaker(), genDb, 0) failAt = rand.Int() % len(headers) failNum = headers[failAt].Number.Uint64() @@ -734,7 +737,6 @@ func testInsertNonceError(t *testing.T, full bool) { func TestFastVsFullChains(t *testing.T) { // Configure and generate a sample block chain var ( - gendb = rawdb.NewMemoryDatabase() key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") address = crypto.PubkeyToAddress(key.PublicKey) funds = big.NewInt(1000000000000000) @@ -743,10 +745,9 @@ func TestFastVsFullChains(t *testing.T) { Alloc: GenesisAlloc{address: {Balance: funds}}, BaseFee: big.NewInt(params.InitialBaseFee), } - genesis = gspec.MustCommit(gendb) - signer = types.LatestSigner(gspec.Config) + signer = types.LatestSigner(gspec.Config) ) - blocks, receipts := GenerateChain(gspec.Config, genesis, ethash.NewFaker(), gendb, 1024, func(i int, block *BlockGen) { + _, blocks, receipts := GenerateChainWithGenesis(gspec, ethash.NewFaker(), 1024, func(i int, block *BlockGen) { block.SetCoinbase(common.Address{0x00}) // If the block number is multiple of 3, send a few bonus transactions to the miner @@ -759,15 +760,14 @@ func TestFastVsFullChains(t *testing.T) { block.AddTx(tx) } } - // If the block number is a multiple of 5, add a few bonus uncles to the block - if i%5 == 5 { - block.AddUncle(&types.Header{ParentHash: block.PrevBlock(i - 1).Hash(), Number: big.NewInt(int64(i - 1))}) + // If the block number is a multiple of 5, add an uncle to the block + if i%5 == 4 { + block.AddUncle(&types.Header{ParentHash: block.PrevBlock(i - 2).Hash(), Number: big.NewInt(int64(i))}) } }) // Import the chain as an archive node for the comparison baseline archiveDb := rawdb.NewMemoryDatabase() - gspec.MustCommit(archiveDb) - archive, _ := NewBlockChain(archiveDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) + archive, _ := NewBlockChain(archiveDb, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) defer archive.Stop() if n, err := archive.InsertChain(blocks); err != nil { @@ -775,8 +775,7 @@ func TestFastVsFullChains(t *testing.T) { } // Fast import the chain as a non-archive node to test fastDb := rawdb.NewMemoryDatabase() - gspec.MustCommit(fastDb) - fast, _ := NewBlockChain(fastDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) + fast, _ := NewBlockChain(fastDb, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) defer fast.Stop() headers := make([]*types.Header, len(blocks)) @@ -790,14 +789,12 @@ func TestFastVsFullChains(t *testing.T) { t.Fatalf("failed to insert receipt %d: %v", n, err) } // Freezer style fast import the chain. - frdir := t.TempDir() - ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "", false) + ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false) if err != nil { t.Fatalf("failed to create temp freezer db: %v", err) } defer ancientDb.Close() - gspec.MustCommit(ancientDb) - ancient, _ := NewBlockChain(ancientDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) + ancient, _ := NewBlockChain(ancientDb, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) defer ancient.Stop() if n, err := ancient.InsertHeaderChain(headers, 1); err != nil { @@ -867,7 +864,6 @@ func TestFastVsFullChains(t *testing.T) { func TestLightVsFastVsFullChainHeads(t *testing.T) { // Configure and generate a sample block chain var ( - gendb = rawdb.NewMemoryDatabase() key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") address = crypto.PubkeyToAddress(key.PublicKey) funds = big.NewInt(1000000000000000) @@ -876,19 +872,16 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) { Alloc: GenesisAlloc{address: {Balance: funds}}, BaseFee: big.NewInt(params.InitialBaseFee), } - genesis = gspec.MustCommit(gendb) ) height := uint64(1024) - blocks, receipts := GenerateChain(gspec.Config, genesis, ethash.NewFaker(), gendb, int(height), nil) + _, blocks, receipts := GenerateChainWithGenesis(gspec, ethash.NewFaker(), int(height), nil) // makeDb creates a db instance for testing. makeDb := func() ethdb.Database { - dir := t.TempDir() - db, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), dir, "", false) + db, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false) if err != nil { t.Fatalf("failed to create temp freezer db: %v", err) } - gspec.MustCommit(db) return db } // Configure a subchain to roll back @@ -915,7 +908,7 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) { archiveCaching := *defaultCacheConfig archiveCaching.TrieDirtyDisabled = true - archive, _ := NewBlockChain(archiveDb, &archiveCaching, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) + archive, _ := NewBlockChain(archiveDb, &archiveCaching, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) if n, err := archive.InsertChain(blocks); err != nil { t.Fatalf("failed to process block %d: %v", n, err) } @@ -928,7 +921,7 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) { // Import the chain as a non-archive node and ensure all pointers are updated fastDb := makeDb() defer fastDb.Close() - fast, _ := NewBlockChain(fastDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) + fast, _ := NewBlockChain(fastDb, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) defer fast.Stop() headers := make([]*types.Header, len(blocks)) @@ -948,7 +941,7 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) { // Import the chain as a ancient-first node and ensure all pointers are updated ancientDb := makeDb() defer ancientDb.Close() - ancient, _ := NewBlockChain(ancientDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) + ancient, _ := NewBlockChain(ancientDb, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) defer ancient.Stop() if n, err := ancient.InsertHeaderChain(headers, 1); err != nil { @@ -967,7 +960,7 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) { // Import the chain as a light node and ensure all pointers are updated lightDb := makeDb() defer lightDb.Close() - light, _ := NewBlockChain(lightDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) + light, _ := NewBlockChain(lightDb, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) if n, err := light.InsertHeaderChain(headers, 1); err != nil { t.Fatalf("failed to insert header %d: %v", n, err) } @@ -987,7 +980,6 @@ func TestChainTxReorgs(t *testing.T) { addr1 = crypto.PubkeyToAddress(key1.PublicKey) addr2 = crypto.PubkeyToAddress(key2.PublicKey) addr3 = crypto.PubkeyToAddress(key3.PublicKey) - db = rawdb.NewMemoryDatabase() gspec = &Genesis{ Config: params.TestChainConfig, GasLimit: 3141592, @@ -997,8 +989,7 @@ func TestChainTxReorgs(t *testing.T) { addr3: {Balance: big.NewInt(1000000000000000)}, }, } - genesis = gspec.MustCommit(db) - signer = types.LatestSigner(gspec.Config) + signer = types.LatestSigner(gspec.Config) ) // Create two transactions shared between the chains: @@ -1018,7 +1009,7 @@ func TestChainTxReorgs(t *testing.T) { // - futureAdd: transaction added after the reorg has already finished var pastAdd, freshAdd, futureAdd *types.Transaction - chain, _ := GenerateChain(gspec.Config, genesis, ethash.NewFaker(), db, 3, func(i int, gen *BlockGen) { + _, chain, _ := GenerateChainWithGenesis(gspec, ethash.NewFaker(), 3, func(i int, gen *BlockGen) { switch i { case 0: pastDrop, _ = types.SignTx(types.NewTransaction(gen.TxNonce(addr2), addr2, big.NewInt(1000), params.TxGas, gen.header.BaseFee, nil), signer, key2) @@ -1036,14 +1027,15 @@ func TestChainTxReorgs(t *testing.T) { } }) // Import the chain. This runs all block validation rules. - blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) + db := rawdb.NewMemoryDatabase() + blockchain, _ := NewBlockChain(db, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) if i, err := blockchain.InsertChain(chain); err != nil { t.Fatalf("failed to insert original chain[%d]: %v", i, err) } defer blockchain.Stop() // overwrite the old chain - chain, _ = GenerateChain(gspec.Config, genesis, ethash.NewFaker(), db, 5, func(i int, gen *BlockGen) { + _, chain, _ = GenerateChainWithGenesis(gspec, ethash.NewFaker(), 5, func(i int, gen *BlockGen) { switch i { case 0: pastAdd, _ = types.SignTx(types.NewTransaction(gen.TxNonce(addr3), addr3, big.NewInt(1000), params.TxGas, gen.header.BaseFee, nil), signer, key3) @@ -1098,20 +1090,19 @@ func TestLogReorgs(t *testing.T) { var ( key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") addr1 = crypto.PubkeyToAddress(key1.PublicKey) - db = rawdb.NewMemoryDatabase() + // this code generates a log - code = common.Hex2Bytes("60606040525b7f24ec1d3ff24c2f6ff210738839dbc339cd45a5294d85c79361016243157aae7b60405180905060405180910390a15b600a8060416000396000f360606040526008565b00") - gspec = &Genesis{Config: params.TestChainConfig, Alloc: GenesisAlloc{addr1: {Balance: big.NewInt(10000000000000000)}}} - genesis = gspec.MustCommit(db) - signer = types.LatestSigner(gspec.Config) + code = common.Hex2Bytes("60606040525b7f24ec1d3ff24c2f6ff210738839dbc339cd45a5294d85c79361016243157aae7b60405180905060405180910390a15b600a8060416000396000f360606040526008565b00") + gspec = &Genesis{Config: params.TestChainConfig, Alloc: GenesisAlloc{addr1: {Balance: big.NewInt(10000000000000000)}}} + signer = types.LatestSigner(gspec.Config) ) - blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) + blockchain, _ := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) defer blockchain.Stop() rmLogsCh := make(chan RemovedLogsEvent) blockchain.SubscribeRemovedLogsEvent(rmLogsCh) - chain, _ := GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), db, 2, func(i int, gen *BlockGen) { + _, chain, _ := GenerateChainWithGenesis(gspec, ethash.NewFaker(), 2, func(i int, gen *BlockGen) { if i == 1 { tx, err := types.SignTx(types.NewContractCreation(gen.TxNonce(addr1), new(big.Int), 1000000, gen.header.BaseFee, code), signer, key1) if err != nil { @@ -1124,7 +1115,7 @@ func TestLogReorgs(t *testing.T) { t.Fatalf("failed to insert chain: %v", err) } - chain, _ = GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), db, 3, func(i int, gen *BlockGen) {}) + _, chain, _ = GenerateChainWithGenesis(gspec, ethash.NewFaker(), 3, func(i int, gen *BlockGen) {}) done := make(chan struct{}) go func() { ev := <-rmLogsCh @@ -1154,14 +1145,11 @@ func TestLogRebirth(t *testing.T) { var ( key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") addr1 = crypto.PubkeyToAddress(key1.PublicKey) - db = rawdb.NewMemoryDatabase() gspec = &Genesis{Config: params.TestChainConfig, Alloc: GenesisAlloc{addr1: {Balance: big.NewInt(10000000000000000)}}} - genesis = gspec.MustCommit(db) signer = types.LatestSigner(gspec.Config) engine = ethash.NewFaker() - blockchain, _ = NewBlockChain(db, nil, gspec.Config, engine, vm.Config{}, nil, nil) + blockchain, _ = NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, engine, vm.Config{}, nil, nil) ) - defer blockchain.Stop() // The event channels. @@ -1170,46 +1158,62 @@ func TestLogRebirth(t *testing.T) { blockchain.SubscribeLogsEvent(newLogCh) blockchain.SubscribeRemovedLogsEvent(rmLogsCh) - // This chain contains a single log. - chain, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 2, func(i int, gen *BlockGen) { - if i == 1 { - tx, err := types.SignTx(types.NewContractCreation(gen.TxNonce(addr1), new(big.Int), 1000000, gen.header.BaseFee, logCode), signer, key1) - if err != nil { - t.Fatalf("failed to create tx: %v", err) + // This chain contains 10 logs. + genDb, chain, _ := GenerateChainWithGenesis(gspec, engine, 3, func(i int, gen *BlockGen) { + if i < 2 { + for ii := 0; ii < 5; ii++ { + tx, err := types.SignNewTx(key1, signer, &types.LegacyTx{ + Nonce: gen.TxNonce(addr1), + GasPrice: gen.header.BaseFee, + Gas: uint64(1000001), + Data: logCode, + }) + if err != nil { + t.Fatalf("failed to create tx: %v", err) + } + gen.AddTx(tx) } - gen.AddTx(tx) } }) if _, err := blockchain.InsertChain(chain); err != nil { t.Fatalf("failed to insert chain: %v", err) } - checkLogEvents(t, newLogCh, rmLogsCh, 1, 0) + checkLogEvents(t, newLogCh, rmLogsCh, 10, 0) - // Generate long reorg chain containing another log. Inserting the - // chain removes one log and adds one. - forkChain, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 2, func(i int, gen *BlockGen) { - if i == 1 { - tx, err := types.SignTx(types.NewContractCreation(gen.TxNonce(addr1), new(big.Int), 1000000, gen.header.BaseFee, logCode), signer, key1) + // Generate long reorg chain containing more logs. Inserting the + // chain removes one log and adds four. + _, forkChain, _ := GenerateChainWithGenesis(gspec, engine, 3, func(i int, gen *BlockGen) { + if i == 2 { + // The last (head) block is not part of the reorg-chain, we can ignore it + return + } + for ii := 0; ii < 5; ii++ { + tx, err := types.SignNewTx(key1, signer, &types.LegacyTx{ + Nonce: gen.TxNonce(addr1), + GasPrice: gen.header.BaseFee, + Gas: uint64(1000000), + Data: logCode, + }) if err != nil { t.Fatalf("failed to create tx: %v", err) } gen.AddTx(tx) - gen.OffsetTime(-9) // higher block difficulty } + gen.OffsetTime(-9) // higher block difficulty }) if _, err := blockchain.InsertChain(forkChain); err != nil { t.Fatalf("failed to insert forked chain: %v", err) } - checkLogEvents(t, newLogCh, rmLogsCh, 1, 1) + checkLogEvents(t, newLogCh, rmLogsCh, 10, 10) // This chain segment is rooted in the original chain, but doesn't contain any logs. // When inserting it, the canonical chain switches away from forkChain and re-emits // the log event for the old chain, as well as a RemovedLogsEvent for forkChain. - newBlocks, _ := GenerateChain(params.TestChainConfig, chain[len(chain)-1], engine, db, 1, func(i int, gen *BlockGen) {}) + newBlocks, _ := GenerateChain(gspec.Config, chain[len(chain)-1], engine, genDb, 1, func(i int, gen *BlockGen) {}) if _, err := blockchain.InsertChain(newBlocks); err != nil { t.Fatalf("failed to insert forked chain: %v", err) } - checkLogEvents(t, newLogCh, rmLogsCh, 1, 1) + checkLogEvents(t, newLogCh, rmLogsCh, 10, 10) } // This test is a variation of TestLogRebirth. It verifies that log events are emitted @@ -1218,13 +1222,10 @@ func TestSideLogRebirth(t *testing.T) { var ( key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") addr1 = crypto.PubkeyToAddress(key1.PublicKey) - db = rawdb.NewMemoryDatabase() gspec = &Genesis{Config: params.TestChainConfig, Alloc: GenesisAlloc{addr1: {Balance: big.NewInt(10000000000000000)}}} - genesis = gspec.MustCommit(db) signer = types.LatestSigner(gspec.Config) - blockchain, _ = NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) + blockchain, _ = NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) ) - defer blockchain.Stop() newLogCh := make(chan []*types.Log, 10) @@ -1232,10 +1233,9 @@ func TestSideLogRebirth(t *testing.T) { blockchain.SubscribeLogsEvent(newLogCh) blockchain.SubscribeRemovedLogsEvent(rmLogsCh) - chain, _ := GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), db, 2, func(i int, gen *BlockGen) { + _, chain, _ := GenerateChainWithGenesis(gspec, ethash.NewFaker(), 2, func(i int, gen *BlockGen) { if i == 1 { gen.OffsetTime(-9) // higher block difficulty - } }) if _, err := blockchain.InsertChain(chain); err != nil { @@ -1244,7 +1244,7 @@ func TestSideLogRebirth(t *testing.T) { checkLogEvents(t, newLogCh, rmLogsCh, 0, 0) // Generate side chain with lower difficulty - sideChain, _ := GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), db, 2, func(i int, gen *BlockGen) { + genDb, sideChain, _ := GenerateChainWithGenesis(gspec, ethash.NewFaker(), 2, func(i int, gen *BlockGen) { if i == 1 { tx, err := types.SignTx(types.NewContractCreation(gen.TxNonce(addr1), new(big.Int), 1000000, gen.header.BaseFee, logCode), signer, key1) if err != nil { @@ -1259,7 +1259,7 @@ func TestSideLogRebirth(t *testing.T) { checkLogEvents(t, newLogCh, rmLogsCh, 0, 0) // Generate a new block based on side chain. - newBlocks, _ := GenerateChain(params.TestChainConfig, sideChain[len(sideChain)-1], ethash.NewFaker(), db, 1, func(i int, gen *BlockGen) {}) + newBlocks, _ := GenerateChain(gspec.Config, sideChain[len(sideChain)-1], ethash.NewFaker(), genDb, 1, func(i int, gen *BlockGen) {}) if _, err := blockchain.InsertChain(newBlocks); err != nil { t.Fatalf("failed to insert forked chain: %v", err) } @@ -1268,44 +1268,65 @@ func TestSideLogRebirth(t *testing.T) { func checkLogEvents(t *testing.T, logsCh <-chan []*types.Log, rmLogsCh <-chan RemovedLogsEvent, wantNew, wantRemoved int) { t.Helper() - - if len(logsCh) != wantNew { - t.Fatalf("wrong number of log events: got %d, want %d", len(logsCh), wantNew) - } - if len(rmLogsCh) != wantRemoved { - t.Fatalf("wrong number of removed log events: got %d, want %d", len(rmLogsCh), wantRemoved) - } + var ( + countNew int + countRm int + prev int + ) // Drain events. - for i := 0; i < len(logsCh); i++ { - <-logsCh + for len(logsCh) > 0 { + x := <-logsCh + countNew += len(x) + for _, log := range x { + // We expect added logs to be in ascending order: 0:0, 0:1, 1:0 ... + have := 100*int(log.BlockNumber) + int(log.TxIndex) + if have < prev { + t.Fatalf("Expected new logs to arrive in ascending order (%d < %d)", have, prev) + } + prev = have + } + } + prev = 0 + for len(rmLogsCh) > 0 { + x := <-rmLogsCh + countRm += len(x.Logs) + for _, log := range x.Logs { + // We expect removed logs to be in ascending order: 0:0, 0:1, 1:0 ... + have := 100*int(log.BlockNumber) + int(log.TxIndex) + if have < prev { + t.Fatalf("Expected removed logs to arrive in ascending order (%d < %d)", have, prev) + } + prev = have + } + } + + if countNew != wantNew { + t.Fatalf("wrong number of log events: got %d, want %d", countNew, wantNew) } - for i := 0; i < len(rmLogsCh); i++ { - <-rmLogsCh + if countRm != wantRemoved { + t.Fatalf("wrong number of removed log events: got %d, want %d", countRm, wantRemoved) } } func TestReorgSideEvent(t *testing.T) { var ( - db = rawdb.NewMemoryDatabase() key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") addr1 = crypto.PubkeyToAddress(key1.PublicKey) gspec = &Genesis{ Config: params.TestChainConfig, Alloc: GenesisAlloc{addr1: {Balance: big.NewInt(10000000000000000)}}, } - genesis = gspec.MustCommit(db) - signer = types.LatestSigner(gspec.Config) + signer = types.LatestSigner(gspec.Config) ) - - blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) + blockchain, _ := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) defer blockchain.Stop() - chain, _ := GenerateChain(gspec.Config, genesis, ethash.NewFaker(), db, 3, func(i int, gen *BlockGen) {}) + _, chain, _ := GenerateChainWithGenesis(gspec, ethash.NewFaker(), 3, func(i int, gen *BlockGen) {}) if _, err := blockchain.InsertChain(chain); err != nil { t.Fatalf("failed to insert chain: %v", err) } - replacementBlocks, _ := GenerateChain(gspec.Config, genesis, ethash.NewFaker(), db, 4, func(i int, gen *BlockGen) { + _, replacementBlocks, _ := GenerateChainWithGenesis(gspec, ethash.NewFaker(), 4, func(i int, gen *BlockGen) { tx, err := types.SignTx(types.NewContractCreation(gen.TxNonce(addr1), new(big.Int), 1000000, gen.header.BaseFee, nil), signer, key1) if i == 2 { gen.OffsetTime(-9) @@ -1364,18 +1385,17 @@ done: t.Errorf("unexpected event fired: %v", e) case <-time.After(250 * time.Millisecond): } - } // Tests if the canonical block can be fetched from the database during chain insertion. func TestCanonicalBlockRetrieval(t *testing.T) { - _, blockchain, err := newCanonical(ethash.NewFaker(), 0, true) + _, gspec, blockchain, err := newCanonical(ethash.NewFaker(), 0, true) if err != nil { t.Fatalf("failed to create pristine chain: %v", err) } defer blockchain.Stop() - chain, _ := GenerateChain(blockchain.chainConfig, blockchain.genesisBlock, ethash.NewFaker(), blockchain.db, 10, func(i int, gen *BlockGen) {}) + _, chain, _ := GenerateChainWithGenesis(gspec, ethash.NewFaker(), 10, func(i int, gen *BlockGen) {}) var pend sync.WaitGroup pend.Add(len(chain)) @@ -1417,22 +1437,21 @@ func TestCanonicalBlockRetrieval(t *testing.T) { func TestEIP155Transition(t *testing.T) { // Configure and generate a sample block chain var ( - db = rawdb.NewMemoryDatabase() key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") address = crypto.PubkeyToAddress(key.PublicKey) funds = big.NewInt(1000000000) deleteAddr = common.Address{1} gspec = &Genesis{ - Config: ¶ms.ChainConfig{ChainID: big.NewInt(1), EIP150Block: big.NewInt(0), EIP155Block: big.NewInt(2), HomesteadBlock: new(big.Int)}, - Alloc: GenesisAlloc{address: {Balance: funds}, deleteAddr: {Balance: new(big.Int)}}, + Config: ¶ms.ChainConfig{ + ChainID: big.NewInt(1), + EIP150Block: big.NewInt(0), + EIP155Block: big.NewInt(2), + HomesteadBlock: new(big.Int), + }, + Alloc: GenesisAlloc{address: {Balance: funds}, deleteAddr: {Balance: new(big.Int)}}, } - genesis = gspec.MustCommit(db) ) - - blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) - defer blockchain.Stop() - - blocks, _ := GenerateChain(gspec.Config, genesis, ethash.NewFaker(), db, 4, func(i int, block *BlockGen) { + genDb, blocks, _ := GenerateChainWithGenesis(gspec, ethash.NewFaker(), 4, func(i int, block *BlockGen) { var ( tx *types.Transaction err error @@ -1474,6 +1493,9 @@ func TestEIP155Transition(t *testing.T) { } }) + blockchain, _ := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + defer blockchain.Stop() + if _, err := blockchain.InsertChain(blocks); err != nil { t.Fatal(err) } @@ -1494,8 +1516,13 @@ func TestEIP155Transition(t *testing.T) { } // generate an invalid chain id transaction - config := ¶ms.ChainConfig{ChainID: big.NewInt(2), EIP150Block: big.NewInt(0), EIP155Block: big.NewInt(2), HomesteadBlock: new(big.Int)} - blocks, _ = GenerateChain(config, blocks[len(blocks)-1], ethash.NewFaker(), db, 4, func(i int, block *BlockGen) { + config := ¶ms.ChainConfig{ + ChainID: big.NewInt(2), + EIP150Block: big.NewInt(0), + EIP155Block: big.NewInt(2), + HomesteadBlock: new(big.Int), + } + blocks, _ = GenerateChain(config, blocks[len(blocks)-1], ethash.NewFaker(), genDb, 4, func(i int, block *BlockGen) { var ( tx *types.Transaction err error @@ -1520,7 +1547,6 @@ func TestEIP155Transition(t *testing.T) { func TestEIP161AccountRemoval(t *testing.T) { // Configure and generate a sample block chain var ( - db = rawdb.NewMemoryDatabase() key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") address = crypto.PubkeyToAddress(key.PublicKey) funds = big.NewInt(1000000000) @@ -1535,12 +1561,8 @@ func TestEIP161AccountRemoval(t *testing.T) { }, Alloc: GenesisAlloc{address: {Balance: funds}}, } - genesis = gspec.MustCommit(db) ) - blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) - defer blockchain.Stop() - - blocks, _ := GenerateChain(gspec.Config, genesis, ethash.NewFaker(), db, 3, func(i int, block *BlockGen) { + _, blocks, _ := GenerateChainWithGenesis(gspec, ethash.NewFaker(), 3, func(i int, block *BlockGen) { var ( tx *types.Transaction err error @@ -1560,6 +1582,9 @@ func TestEIP161AccountRemoval(t *testing.T) { block.AddTx(tx) }) // account must exist pre eip 161 + blockchain, _ := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + defer blockchain.Stop() + if _, err := blockchain.InsertChain(types.Blocks{blocks[0]}); err != nil { t.Fatal(err) } @@ -1592,27 +1617,25 @@ func TestEIP161AccountRemoval(t *testing.T) { func TestBlockchainHeaderchainReorgConsistency(t *testing.T) { // Generate a canonical chain to act as the main dataset engine := ethash.NewFaker() - - db := rawdb.NewMemoryDatabase() - genesis := (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(db) - blocks, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 64, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) }) + genesis := &Genesis{ + Config: params.TestChainConfig, + BaseFee: big.NewInt(params.InitialBaseFee), + } + genDb, blocks, _ := GenerateChainWithGenesis(genesis, engine, 64, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) }) // Generate a bunch of fork blocks, each side forking from the canonical chain forks := make([]*types.Block, len(blocks)) for i := 0; i < len(forks); i++ { - parent := genesis + parent := genesis.ToBlock() if i > 0 { parent = blocks[i-1] } - fork, _ := GenerateChain(params.TestChainConfig, parent, engine, db, 1, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{2}) }) + fork, _ := GenerateChain(genesis.Config, parent, engine, genDb, 1, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{2}) }) forks[i] = fork[0] } // Import the canonical and fork chain side by side, verifying the current block // and current header consistency - diskdb := rawdb.NewMemoryDatabase() - (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(diskdb) - - chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil, nil) + chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), nil, genesis, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -1637,26 +1660,24 @@ func TestBlockchainHeaderchainReorgConsistency(t *testing.T) { func TestTrieForkGC(t *testing.T) { // Generate a canonical chain to act as the main dataset engine := ethash.NewFaker() - - db := rawdb.NewMemoryDatabase() - genesis := (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(db) - blocks, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 2*TriesInMemory, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) }) + genesis := &Genesis{ + Config: params.TestChainConfig, + BaseFee: big.NewInt(params.InitialBaseFee), + } + genDb, blocks, _ := GenerateChainWithGenesis(genesis, engine, 2*TriesInMemory, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) }) // Generate a bunch of fork blocks, each side forking from the canonical chain forks := make([]*types.Block, len(blocks)) for i := 0; i < len(forks); i++ { - parent := genesis + parent := genesis.ToBlock() if i > 0 { parent = blocks[i-1] } - fork, _ := GenerateChain(params.TestChainConfig, parent, engine, db, 1, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{2}) }) + fork, _ := GenerateChain(genesis.Config, parent, engine, genDb, 1, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{2}) }) forks[i] = fork[0] } // Import the canonical and fork chain side by side, forcing the trie cache to cache both - diskdb := rawdb.NewMemoryDatabase() - (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(diskdb) - - chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil, nil) + chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), nil, genesis, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -1683,19 +1704,16 @@ func TestTrieForkGC(t *testing.T) { func TestLargeReorgTrieGC(t *testing.T) { // Generate the original common chain segment and the two competing forks engine := ethash.NewFaker() - - db := rawdb.NewMemoryDatabase() - genesis := (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(db) - - shared, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 64, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) }) - original, _ := GenerateChain(params.TestChainConfig, shared[len(shared)-1], engine, db, 2*TriesInMemory, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{2}) }) - competitor, _ := GenerateChain(params.TestChainConfig, shared[len(shared)-1], engine, db, 2*TriesInMemory+1, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{3}) }) + genesis := &Genesis{ + Config: params.TestChainConfig, + BaseFee: big.NewInt(params.InitialBaseFee), + } + genDb, shared, _ := GenerateChainWithGenesis(genesis, engine, 64, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) }) + original, _ := GenerateChain(genesis.Config, shared[len(shared)-1], engine, genDb, 2*TriesInMemory, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{2}) }) + competitor, _ := GenerateChain(genesis.Config, shared[len(shared)-1], engine, genDb, 2*TriesInMemory+1, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{3}) }) // Import the shared chain and the original canonical one - diskdb := rawdb.NewMemoryDatabase() - (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(diskdb) - - chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil, nil) + chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), nil, genesis, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -1734,26 +1752,21 @@ func TestLargeReorgTrieGC(t *testing.T) { func TestBlockchainRecovery(t *testing.T) { // Configure and generate a sample block chain var ( - gendb = rawdb.NewMemoryDatabase() key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") address = crypto.PubkeyToAddress(key.PublicKey) funds = big.NewInt(1000000000) gspec = &Genesis{Config: params.TestChainConfig, Alloc: GenesisAlloc{address: {Balance: funds}}} - genesis = gspec.MustCommit(gendb) ) height := uint64(1024) - blocks, receipts := GenerateChain(gspec.Config, genesis, ethash.NewFaker(), gendb, int(height), nil) + _, blocks, receipts := GenerateChainWithGenesis(gspec, ethash.NewFaker(), int(height), nil) // Import the chain as a ancient-first node and ensure all pointers are updated - frdir := t.TempDir() - - ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "", false) + ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false) if err != nil { t.Fatalf("failed to create temp freezer db: %v", err) } defer ancientDb.Close() - gspec.MustCommit(ancientDb) - ancient, _ := NewBlockChain(ancientDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) + ancient, _ := NewBlockChain(ancientDb, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) headers := make([]*types.Header, len(blocks)) for i, block := range blocks { @@ -1773,7 +1786,7 @@ func TestBlockchainRecovery(t *testing.T) { rawdb.WriteHeadFastBlockHash(ancientDb, midBlock.Hash()) // Reopen broken blockchain again - ancient, _ = NewBlockChain(ancientDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) + ancient, _ = NewBlockChain(ancientDb, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) defer ancient.Stop() if num := ancient.CurrentBlock().NumberU64(); num != 0 { t.Errorf("head block mismatch: have #%v, want #%v", num, 0) @@ -1789,7 +1802,7 @@ func TestBlockchainRecovery(t *testing.T) { // This test checks that InsertReceiptChain will roll back correctly when attempting to insert a side chain. func TestInsertReceiptChainRollback(t *testing.T) { // Generate forked chain. The returned BlockChain object is used to process the side chain blocks. - tmpChain, sideblocks, canonblocks, err := getLongAndShortChains() + tmpChain, sideblocks, canonblocks, gspec, err := getLongAndShortChains() if err != nil { t.Fatal(err) } @@ -1814,15 +1827,13 @@ func TestInsertReceiptChainRollback(t *testing.T) { } // Set up a BlockChain that uses the ancient store. - frdir := t.TempDir() - ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "", false) + ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false) if err != nil { t.Fatalf("failed to create temp freezer db: %v", err) } defer ancientDb.Close() - gspec := Genesis{Config: params.AllEthashProtocolChanges} - gspec.MustCommit(ancientDb) - ancientChain, _ := NewBlockChain(ancientDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) + + ancientChain, _ := NewBlockChain(ancientDb, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) defer ancientChain.Stop() // Import the canonical header chain. @@ -1864,26 +1875,24 @@ func TestInsertReceiptChainRollback(t *testing.T) { // overtake the 'canon' chain until after it's passed canon by about 200 blocks. // // Details at: -// - https://github.com/ethereum/go-ethereum/issues/18977 -// - https://github.com/ethereum/go-ethereum/pull/18988 +// - https://github.com/ethereum/go-ethereum/issues/18977 +// - https://github.com/ethereum/go-ethereum/pull/18988 func TestLowDiffLongChain(t *testing.T) { // Generate a canonical chain to act as the main dataset engine := ethash.NewFaker() - db := rawdb.NewMemoryDatabase() - genesis := (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(db) - + genesis := &Genesis{ + Config: params.TestChainConfig, + BaseFee: big.NewInt(params.InitialBaseFee), + } // We must use a pretty long chain to ensure that the fork doesn't overtake us // until after at least 128 blocks post tip - blocks, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 6*TriesInMemory, func(i int, b *BlockGen) { + genDb, blocks, _ := GenerateChainWithGenesis(genesis, engine, 6*TriesInMemory, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) b.OffsetTime(-9) }) // Import the canonical chain - diskdb := rawdb.NewMemoryDatabase() - (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(diskdb) - - chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil, nil) + chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), nil, genesis, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -1892,7 +1901,7 @@ func TestLowDiffLongChain(t *testing.T) { } // Generate fork chain, starting from an early block parent := blocks[10] - fork, _ := GenerateChain(params.TestChainConfig, parent, engine, db, 8*TriesInMemory, func(i int, b *BlockGen) { + fork, _ := GenerateChain(genesis.Config, parent, engine, genDb, 8*TriesInMemory, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{2}) }) @@ -1925,15 +1934,11 @@ func TestLowDiffLongChain(t *testing.T) { // 0: the transition happens since genesis // 1: the transition happens after some chain segments func testSideImport(t *testing.T, numCanonBlocksInSidechain, blocksBetweenCommonAncestorAndPruneblock int, mergePoint int) { - // Copy the TestChainConfig so we can modify it during tests - chainConfig := *params.TestChainConfig // Generate a canonical chain to act as the main dataset + chainConfig := *params.TestChainConfig var ( - merger = consensus.NewMerger(rawdb.NewMemoryDatabase()) - genEngine = beacon.New(ethash.NewFaker()) - runEngine = beacon.New(ethash.NewFaker()) - db = rawdb.NewMemoryDatabase() - + merger = consensus.NewMerger(rawdb.NewMemoryDatabase()) + engine = beacon.New(ethash.NewFaker()) key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") addr = crypto.PubkeyToAddress(key.PublicKey) nonce = uint64(0) @@ -1943,13 +1948,10 @@ func testSideImport(t *testing.T, numCanonBlocksInSidechain, blocksBetweenCommon Alloc: GenesisAlloc{addr: {Balance: big.NewInt(math.MaxInt64)}}, BaseFee: big.NewInt(params.InitialBaseFee), } - signer = types.LatestSigner(gspec.Config) - genesis, _ = gspec.Commit(db) + signer = types.LatestSigner(gspec.Config) ) // Generate and import the canonical chain - diskdb := rawdb.NewMemoryDatabase() - gspec.MustCommit(diskdb) - chain, err := NewBlockChain(diskdb, nil, &chainConfig, runEngine, vm.Config{}, nil, nil) + chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -1961,7 +1963,7 @@ func testSideImport(t *testing.T, numCanonBlocksInSidechain, blocksBetweenCommon // Set the terminal total difficulty in the config gspec.Config.TerminalTotalDifficulty = big.NewInt(0) } - blocks, _ := GenerateChain(&chainConfig, genesis, genEngine, db, 2*TriesInMemory, func(i int, gen *BlockGen) { + genDb, blocks, _ := GenerateChainWithGenesis(gspec, engine, 2*TriesInMemory, func(i int, gen *BlockGen) { tx, err := types.SignTx(types.NewTransaction(nonce, common.HexToAddress("deadbeef"), big.NewInt(100), 21000, big.NewInt(int64(i+1)*params.GWei), nil), signer, key) if err != nil { t.Fatalf("failed to create tx: %v", err) @@ -2001,7 +2003,7 @@ func testSideImport(t *testing.T, numCanonBlocksInSidechain, blocksBetweenCommon // Generate fork chain, make it longer than canon parentIndex := lastPrunedIndex + blocksBetweenCommonAncestorAndPruneblock parent := blocks[parentIndex] - fork, _ := GenerateChain(&chainConfig, parent, genEngine, db, 2*TriesInMemory, func(i int, b *BlockGen) { + fork, _ := GenerateChain(gspec.Config, parent, engine, genDb, 2*TriesInMemory, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{2}) }) // Prepend the parent(s) @@ -2021,14 +2023,16 @@ func testSideImport(t *testing.T, numCanonBlocksInSidechain, blocksBetweenCommon } // Tests that importing a sidechain (S), where -// - S is sidechain, containing blocks [Sn...Sm] -// - C is canon chain, containing blocks [G..Cn..Cm] -// - The common ancestor Cc is pruned -// - The first block in S: Sn, is == Cn +// - S is sidechain, containing blocks [Sn...Sm] +// - C is canon chain, containing blocks [G..Cn..Cm] +// - The common ancestor Cc is pruned +// - The first block in S: Sn, is == Cn +// // That is: the sidechain for import contains some blocks already present in canon chain. -// So the blocks are -// [ Cn, Cn+1, Cc, Sn+3 ... Sm] -// ^ ^ ^ pruned +// So the blocks are: +// +// [ Cn, Cn+1, Cc, Sn+3 ... Sm] +// ^ ^ ^ pruned func TestPrunedImportSide(t *testing.T) { //glogger := log.NewGlogHandler(log.StreamHandler(os.Stdout, log.TerminalFormat(false))) //glogger.Verbosity(3) @@ -2063,28 +2067,28 @@ func TestInsertKnownBlocks(t *testing.T) { testInsertKnownChainData(t, "bl func testInsertKnownChainData(t *testing.T, typ string) { engine := ethash.NewFaker() + genesis := &Genesis{ + Config: params.TestChainConfig, + BaseFee: big.NewInt(params.InitialBaseFee), + } + genDb, blocks, receipts := GenerateChainWithGenesis(genesis, engine, 32, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) }) - db := rawdb.NewMemoryDatabase() - genesis := (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(db) - - blocks, receipts := GenerateChain(params.TestChainConfig, genesis, engine, db, 32, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) }) // A longer chain but total difficulty is lower. - blocks2, receipts2 := GenerateChain(params.TestChainConfig, blocks[len(blocks)-1], engine, db, 65, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) }) + blocks2, receipts2 := GenerateChain(genesis.Config, blocks[len(blocks)-1], engine, genDb, 65, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) }) + // A shorter chain but total difficulty is higher. - blocks3, receipts3 := GenerateChain(params.TestChainConfig, blocks[len(blocks)-1], engine, db, 64, func(i int, b *BlockGen) { + blocks3, receipts3 := GenerateChain(genesis.Config, blocks[len(blocks)-1], engine, genDb, 64, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) b.OffsetTime(-9) // A higher difficulty }) // Import the shared chain and the original canonical one - dir := t.TempDir() - chaindb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), dir, "", false) + chaindb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false) if err != nil { t.Fatalf("failed to create temp freezer db: %v", err) } - (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(chaindb) defer chaindb.Close() - chain, err := NewBlockChain(chaindb, nil, params.TestChainConfig, engine, vm.Config{}, nil, nil) + chain, err := NewBlockChain(chaindb, nil, genesis, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -2204,47 +2208,37 @@ func testInsertKnownChainDataWithMerging(t *testing.T, typ string, mergeHeight i // Copy the TestChainConfig so we can modify it during tests chainConfig := *params.TestChainConfig var ( - db = rawdb.NewMemoryDatabase() - genesis = (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee), Config: &chainConfig}).MustCommit(db) - runMerger = consensus.NewMerger(db) - runEngine = beacon.New(ethash.NewFaker()) - genEngine = beacon.New(ethash.NewFaker()) - ) - applyMerge := func(engine *beacon.Beacon, height int) { - if engine != nil { - runMerger.FinalizePoS() - // Set the terminal total difficulty in the config - chainConfig.TerminalTotalDifficulty = big.NewInt(int64(height)) + genesis = &Genesis{ + BaseFee: big.NewInt(params.InitialBaseFee), + Config: &chainConfig, } - } - + engine = beacon.New(ethash.NewFaker()) + ) // Apply merging since genesis if mergeHeight == 0 { - applyMerge(genEngine, 0) + genesis.Config.TerminalTotalDifficulty = big.NewInt(0) } - blocks, receipts := GenerateChain(&chainConfig, genesis, genEngine, db, 32, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) }) + genDb, blocks, receipts := GenerateChainWithGenesis(genesis, engine, 32, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) }) // Apply merging after the first segment if mergeHeight == 1 { - applyMerge(genEngine, len(blocks)) + genesis.Config.TerminalTotalDifficulty = big.NewInt(int64(len(blocks))) } // Longer chain and shorter chain - blocks2, receipts2 := GenerateChain(&chainConfig, blocks[len(blocks)-1], genEngine, db, 65, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) }) - blocks3, receipts3 := GenerateChain(&chainConfig, blocks[len(blocks)-1], genEngine, db, 64, func(i int, b *BlockGen) { + blocks2, receipts2 := GenerateChain(genesis.Config, blocks[len(blocks)-1], engine, genDb, 65, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) }) + blocks3, receipts3 := GenerateChain(genesis.Config, blocks[len(blocks)-1], engine, genDb, 64, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) b.OffsetTime(-9) // Time shifted, difficulty shouldn't be changed }) // Import the shared chain and the original canonical one - dir := t.TempDir() - chaindb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), dir, "", false) + chaindb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false) if err != nil { t.Fatalf("failed to create temp freezer db: %v", err) } - (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(chaindb) defer chaindb.Close() - chain, err := NewBlockChain(chaindb, nil, &chainConfig, runEngine, vm.Config{}, nil, nil) + chain, err := NewBlockChain(chaindb, nil, genesis, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -2295,11 +2289,6 @@ func testInsertKnownChainDataWithMerging(t *testing.T, typ string, mergeHeight i } } } - - // Apply merging since genesis if required - if mergeHeight == 0 { - applyMerge(runEngine, 0) - } if err := inserter(blocks, receipts); err != nil { t.Fatalf("failed to insert chain data: %v", err) } @@ -2319,11 +2308,6 @@ func testInsertKnownChainDataWithMerging(t *testing.T, typ string, mergeHeight i } asserter(t, blocks[len(blocks)-1]) - // Apply merging after the first segment - if mergeHeight == 1 { - applyMerge(runEngine, len(blocks)) - } - // Import a longer chain with some known data as prefix. if err := inserter(append(blocks, blocks2...), append(receipts, receipts2...)); err != nil { t.Fatalf("failed to insert chain data: %v", err) @@ -2348,32 +2332,30 @@ func testInsertKnownChainDataWithMerging(t *testing.T, typ string, mergeHeight i } // getLongAndShortChains returns two chains: A is longer, B is heavier. -func getLongAndShortChains() (bc *BlockChain, longChain []*types.Block, heavyChain []*types.Block, err error) { +func getLongAndShortChains() (*BlockChain, []*types.Block, []*types.Block, *Genesis, error) { // Generate a canonical chain to act as the main dataset engine := ethash.NewFaker() - db := rawdb.NewMemoryDatabase() - genesis := (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(db) - + genesis := &Genesis{ + Config: params.TestChainConfig, + BaseFee: big.NewInt(params.InitialBaseFee), + } // Generate and import the canonical chain, // Offset the time, to keep the difficulty low - longChain, _ = GenerateChain(params.TestChainConfig, genesis, engine, db, 80, func(i int, b *BlockGen) { + genDb, longChain, _ := GenerateChainWithGenesis(genesis, engine, 80, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) }) - diskdb := rawdb.NewMemoryDatabase() - (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(diskdb) - - chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil, nil) + chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), nil, genesis, nil, engine, vm.Config{}, nil, nil) if err != nil { - return nil, nil, nil, fmt.Errorf("failed to create tester chain: %v", err) + return nil, nil, nil, nil, fmt.Errorf("failed to create tester chain: %v", err) } - // Generate fork chain, make it shorter than canon, with common ancestor pretty early parentIndex := 3 parent := longChain[parentIndex] - heavyChainExt, _ := GenerateChain(params.TestChainConfig, parent, engine, db, 75, func(i int, b *BlockGen) { + heavyChainExt, _ := GenerateChain(genesis.Config, parent, engine, genDb, 75, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{2}) b.OffsetTime(-9) }) + var heavyChain []*types.Block heavyChain = append(heavyChain, longChain[:parentIndex+1]...) heavyChain = append(heavyChain, heavyChainExt...) @@ -2392,14 +2374,14 @@ func getLongAndShortChains() (bc *BlockChain, longChain []*types.Block, heavyCha shorterTd.Add(shorterTd, b.Difficulty()) } if shorterTd.Cmp(longerTd) <= 0 { - return nil, nil, nil, fmt.Errorf("Test is moot, heavyChain td (%v) must be larger than canon td (%v)", shorterTd, longerTd) + return nil, nil, nil, nil, fmt.Errorf("test is moot, heavyChain td (%v) must be larger than canon td (%v)", shorterTd, longerTd) } longerNum := longChain[len(longChain)-1].NumberU64() shorterNum := heavyChain[len(heavyChain)-1].NumberU64() if shorterNum >= longerNum { - return nil, nil, nil, fmt.Errorf("Test is moot, heavyChain num (%v) must be lower than canon num (%v)", shorterNum, longerNum) + return nil, nil, nil, nil, fmt.Errorf("test is moot, heavyChain num (%v) must be lower than canon num (%v)", shorterNum, longerNum) } - return chain, longChain, heavyChain, nil + return chain, longChain, heavyChain, genesis, nil } // TestReorgToShorterRemovesCanonMapping tests that if we @@ -2408,7 +2390,7 @@ func getLongAndShortChains() (bc *BlockChain, longChain []*types.Block, heavyCha // 3. Then there should be no canon mapping for the block at height X // 4. The forked block should still be retrievable by hash func TestReorgToShorterRemovesCanonMapping(t *testing.T) { - chain, canonblocks, sideblocks, err := getLongAndShortChains() + chain, canonblocks, sideblocks, _, err := getLongAndShortChains() if err != nil { t.Fatal(err) } @@ -2444,7 +2426,7 @@ func TestReorgToShorterRemovesCanonMapping(t *testing.T) { // as TestReorgToShorterRemovesCanonMapping, but applied on headerchain // imports -- that is, for fast sync func TestReorgToShorterRemovesCanonMappingHeaderChain(t *testing.T) { - chain, canonblocks, sideblocks, err := getLongAndShortChains() + chain, canonblocks, sideblocks, _, err := getLongAndShortChains() if err != nil { t.Fatal(err) } @@ -2487,7 +2469,6 @@ func TestReorgToShorterRemovesCanonMappingHeaderChain(t *testing.T) { func TestTransactionIndices(t *testing.T) { // Configure and generate a sample block chain var ( - gendb = rawdb.NewMemoryDatabase() key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") address = crypto.PubkeyToAddress(key.PublicKey) funds = big.NewInt(100000000000000000) @@ -2496,18 +2477,15 @@ func TestTransactionIndices(t *testing.T) { Alloc: GenesisAlloc{address: {Balance: funds}}, BaseFee: big.NewInt(params.InitialBaseFee), } - genesis = gspec.MustCommit(gendb) - signer = types.LatestSigner(gspec.Config) + signer = types.LatestSigner(gspec.Config) ) - height := uint64(128) - blocks, receipts := GenerateChain(gspec.Config, genesis, ethash.NewFaker(), gendb, int(height), func(i int, block *BlockGen) { + _, blocks, receipts := GenerateChainWithGenesis(gspec, ethash.NewFaker(), 128, func(i int, block *BlockGen) { tx, err := types.SignTx(types.NewTransaction(block.TxNonce(address), common.Address{0x00}, big.NewInt(1000), params.TxGas, block.header.BaseFee, nil), signer, key) if err != nil { panic(err) } block.AddTx(tx) }) - blocks2, _ := GenerateChain(gspec.Config, blocks[len(blocks)-1], ethash.NewFaker(), gendb, 10, nil) check := func(tail *uint64, chain *BlockChain) { stored := rawdb.ReadTxIndexTail(chain.db) @@ -2542,45 +2520,20 @@ func TestTransactionIndices(t *testing.T) { } } } - frdir := t.TempDir() - ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "", false) - if err != nil { - t.Fatalf("failed to create temp freezer db: %v", err) - } - gspec.MustCommit(ancientDb) - - // Import all blocks into ancient db - l := uint64(0) - chain, err := NewBlockChain(ancientDb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, &l) - if err != nil { - t.Fatalf("failed to create tester chain: %v", err) - } - headers := make([]*types.Header, len(blocks)) - for i, block := range blocks { - headers[i] = block.Header() - } - if n, err := chain.InsertHeaderChain(headers, 0); err != nil { - t.Fatalf("failed to insert header %d: %v", n, err) - } - if n, err := chain.InsertReceiptChain(blocks, receipts, 128); err != nil { - t.Fatalf("block %d: failed to insert into chain: %v", n, err) - } - chain.Stop() - ancientDb.Close() - // Init block chain with external ancients, check all needed indices has been indexed. limit := []uint64{0, 32, 64, 128} for _, l := range limit { - ancientDb, err = rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "", false) - if err != nil { - t.Fatalf("failed to create temp freezer db: %v", err) - } - gspec.MustCommit(ancientDb) - chain, err = NewBlockChain(ancientDb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, &l) + frdir := t.TempDir() + ancientDb, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "", false) + rawdb.WriteAncientBlocks(ancientDb, append([]*types.Block{gspec.ToBlock()}, blocks...), append([]types.Receipts{{}}, receipts...), big.NewInt(0)) + + l := l + chain, err := NewBlockChain(ancientDb, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, &l) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } - time.Sleep(50 * time.Millisecond) // Wait for indices initialisation + chain.indexBlocks(rawdb.ReadTxIndexTail(ancientDb), 128, make(chan struct{})) + var tail uint64 if l != 0 { tail = uint64(128) - l + 1 @@ -2588,26 +2541,27 @@ func TestTransactionIndices(t *testing.T) { check(&tail, chain) chain.Stop() ancientDb.Close() + os.RemoveAll(frdir) } // Reconstruct a block chain which only reserves HEAD-64 tx indices - ancientDb, err = rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "", false) - if err != nil { - t.Fatalf("failed to create temp freezer db: %v", err) - } + ancientDb, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false) defer ancientDb.Close() - gspec.MustCommit(ancientDb) + rawdb.WriteAncientBlocks(ancientDb, append([]*types.Block{gspec.ToBlock()}, blocks...), append([]types.Receipts{{}}, receipts...), big.NewInt(0)) limit = []uint64{0, 64 /* drop stale */, 32 /* shorten history */, 64 /* extend history */, 0 /* restore all */} - tails := []uint64{0, 67 /* 130 - 64 + 1 */, 100 /* 131 - 32 + 1 */, 69 /* 132 - 64 + 1 */, 0} - for i, l := range limit { - chain, err = NewBlockChain(ancientDb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, &l) + for _, l := range limit { + l := l + chain, err := NewBlockChain(ancientDb, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, &l) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } - chain.InsertChain(blocks2[i : i+1]) // Feed chain a higher block to trigger indices updater. - time.Sleep(50 * time.Millisecond) // Wait for indices initialisation - check(&tails[i], chain) + var tail uint64 + if l != 0 { + tail = uint64(128) - l + 1 + } + chain.indexBlocks(rawdb.ReadTxIndexTail(ancientDb), 128, make(chan struct{})) + check(&tail, chain) chain.Stop() } } @@ -2615,16 +2569,13 @@ func TestTransactionIndices(t *testing.T) { func TestSkipStaleTxIndicesInSnapSync(t *testing.T) { // Configure and generate a sample block chain var ( - gendb = rawdb.NewMemoryDatabase() key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") address = crypto.PubkeyToAddress(key.PublicKey) funds = big.NewInt(100000000000000000) gspec = &Genesis{Config: params.TestChainConfig, Alloc: GenesisAlloc{address: {Balance: funds}}} - genesis = gspec.MustCommit(gendb) signer = types.LatestSigner(gspec.Config) ) - height := uint64(128) - blocks, receipts := GenerateChain(gspec.Config, genesis, ethash.NewFaker(), gendb, int(height), func(i int, block *BlockGen) { + _, blocks, receipts := GenerateChainWithGenesis(gspec, ethash.NewFaker(), 128, func(i int, block *BlockGen) { tx, err := types.SignTx(types.NewTransaction(block.TxNonce(address), common.Address{0x00}, big.NewInt(1000), params.TxGas, block.header.BaseFee, nil), signer, key) if err != nil { panic(err) @@ -2666,17 +2617,15 @@ func TestSkipStaleTxIndicesInSnapSync(t *testing.T) { } } - frdir := t.TempDir() - ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "", false) + ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false) if err != nil { t.Fatalf("failed to create temp freezer db: %v", err) } defer ancientDb.Close() - gspec.MustCommit(ancientDb) // Import all blocks into ancient db, only HEAD-32 indices are kept. l := uint64(32) - chain, err := NewBlockChain(ancientDb, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, &l) + chain, err := NewBlockChain(ancientDb, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, &l) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -2702,7 +2651,7 @@ func benchmarkLargeNumberOfValueToNonexisting(b *testing.B, numTxs, numBlocks in testBankKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") testBankAddress = crypto.PubkeyToAddress(testBankKey.PublicKey) bankFunds = big.NewInt(100000000000000000) - gspec = Genesis{ + gspec = &Genesis{ Config: params.TestChainConfig, Alloc: GenesisAlloc{ testBankAddress: {Balance: bankFunds}, @@ -2716,8 +2665,6 @@ func benchmarkLargeNumberOfValueToNonexisting(b *testing.B, numTxs, numBlocks in ) // Generate the original common chain segment and the two competing forks engine := ethash.NewFaker() - db := rawdb.NewMemoryDatabase() - genesis := gspec.MustCommit(db) blockGenerator := func(i int, block *BlockGen) { block.SetCoinbase(common.Address{1}) @@ -2732,15 +2679,12 @@ func benchmarkLargeNumberOfValueToNonexisting(b *testing.B, numTxs, numBlocks in } } - shared, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, numBlocks, blockGenerator) + _, shared, _ := GenerateChainWithGenesis(gspec, engine, numBlocks, blockGenerator) b.StopTimer() b.ResetTimer() for i := 0; i < b.N; i++ { // Import the shared chain and the original canonical one - diskdb := rawdb.NewMemoryDatabase() - gspec.MustCommit(diskdb) - - chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil, nil) + chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, engine, vm.Config{}, nil, nil) if err != nil { b.Fatalf("failed to create tester chain: %v", err) } @@ -2751,7 +2695,6 @@ func benchmarkLargeNumberOfValueToNonexisting(b *testing.B, numTxs, numBlocks in b.StopTimer() if got := chain.CurrentBlock().Transactions().Len(); got != numTxs*numBlocks { b.Fatalf("Transactions were not included, expected %d, got %d", numTxs*numBlocks, got) - } } } @@ -2762,7 +2705,7 @@ func BenchmarkBlockChain_1x1000ValueTransferToNonexisting(b *testing.B) { numBlocks = 1 ) recipientFn := func(nonce uint64) common.Address { - return common.BigToAddress(big.NewInt(0).SetUint64(1337 + nonce)) + return common.BigToAddress(new(big.Int).SetUint64(1337 + nonce)) } dataFn := func(nonce uint64) []byte { return nil @@ -2779,7 +2722,7 @@ func BenchmarkBlockChain_1x1000ValueTransferToExisting(b *testing.B) { b.ResetTimer() recipientFn := func(nonce uint64) common.Address { - return common.BigToAddress(big.NewInt(0).SetUint64(1337)) + return common.BigToAddress(new(big.Int).SetUint64(1337)) } dataFn := func(nonce uint64) []byte { return nil @@ -2796,7 +2739,7 @@ func BenchmarkBlockChain_1x1000Executions(b *testing.B) { b.ResetTimer() recipientFn := func(nonce uint64) common.Address { - return common.BigToAddress(big.NewInt(0).SetUint64(0xc0de)) + return common.BigToAddress(new(big.Int).SetUint64(0xc0de)) } dataFn := func(nonce uint64) []byte { return nil @@ -2809,21 +2752,20 @@ func BenchmarkBlockChain_1x1000Executions(b *testing.B) { // This internally leads to a sidechain import, since the blocks trigger an // ErrPrunedAncestor error. // This may e.g. happen if -// 1. Downloader rollbacks a batch of inserted blocks and exits -// 2. Downloader starts to sync again -// 3. The blocks fetched are all known and canonical blocks +// 1. Downloader rollbacks a batch of inserted blocks and exits +// 2. Downloader starts to sync again +// 3. The blocks fetched are all known and canonical blocks func TestSideImportPrunedBlocks(t *testing.T) { // Generate a canonical chain to act as the main dataset engine := ethash.NewFaker() - db := rawdb.NewMemoryDatabase() - genesis := (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(db) - + genesis := &Genesis{ + Config: params.TestChainConfig, + BaseFee: big.NewInt(params.InitialBaseFee), + } // Generate and import the canonical chain - blocks, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 2*TriesInMemory, nil) - diskdb := rawdb.NewMemoryDatabase() + _, blocks, _ := GenerateChainWithGenesis(genesis, engine, 2*TriesInMemory, nil) - (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(diskdb) - chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil, nil) + chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), nil, genesis, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -2861,11 +2803,9 @@ func TestSideImportPrunedBlocks(t *testing.T) { // first, but the journal wiped the entire state object on create-revert. func TestDeleteCreateRevert(t *testing.T) { var ( - aa = common.HexToAddress("0x000000000000000000000000000000000000aaaa") - bb = common.HexToAddress("0x000000000000000000000000000000000000bbbb") - // Generate a canonical chain to act as the main dataset + aa = common.HexToAddress("0x000000000000000000000000000000000000aaaa") + bb = common.HexToAddress("0x000000000000000000000000000000000000bbbb") engine = ethash.NewFaker() - db = rawdb.NewMemoryDatabase() // A sender who makes transactions, has some funds key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") @@ -2899,10 +2839,9 @@ func TestDeleteCreateRevert(t *testing.T) { }, }, } - genesis = gspec.MustCommit(db) ) - blocks, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 1, func(i int, b *BlockGen) { + _, blocks, _ := GenerateChainWithGenesis(gspec, engine, 1, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) // One transaction to AAAA tx, _ := types.SignTx(types.NewTransaction(0, aa, @@ -2914,10 +2853,7 @@ func TestDeleteCreateRevert(t *testing.T) { b.AddTx(tx) }) // Import the canonical chain - diskdb := rawdb.NewMemoryDatabase() - gspec.MustCommit(diskdb) - - chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil, nil) + chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -2935,9 +2871,8 @@ func TestDeleteCreateRevert(t *testing.T) { // and then the new slots exist func TestDeleteRecreateSlots(t *testing.T) { var ( - // Generate a canonical chain to act as the main dataset engine = ethash.NewFaker() - db = rawdb.NewMemoryDatabase() + // A sender who makes transactions, has some funds key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") address = crypto.PubkeyToAddress(key.PublicKey) @@ -3012,9 +2947,7 @@ func TestDeleteRecreateSlots(t *testing.T) { }, }, } - genesis := gspec.MustCommit(db) - - blocks, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 1, func(i int, b *BlockGen) { + _, blocks, _ := GenerateChainWithGenesis(gspec, engine, 1, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) // One transaction to AA, to kill it tx, _ := types.SignTx(types.NewTransaction(0, aa, @@ -3026,9 +2959,7 @@ func TestDeleteRecreateSlots(t *testing.T) { b.AddTx(tx) }) // Import the canonical chain - diskdb := rawdb.NewMemoryDatabase() - gspec.MustCommit(diskdb) - chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{ + chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, engine, vm.Config{ Debug: true, Tracer: logger.NewJSONLogger(nil, os.Stdout), }, nil, nil) @@ -3062,9 +2993,8 @@ func TestDeleteRecreateSlots(t *testing.T) { // Expected outcome is that _all_ slots are cleared from A func TestDeleteRecreateAccount(t *testing.T) { var ( - // Generate a canonical chain to act as the main dataset engine = ethash.NewFaker() - db = rawdb.NewMemoryDatabase() + // A sender who makes transactions, has some funds key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") address = crypto.PubkeyToAddress(key.PublicKey) @@ -3092,9 +3022,8 @@ func TestDeleteRecreateAccount(t *testing.T) { }, }, } - genesis := gspec.MustCommit(db) - blocks, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 1, func(i int, b *BlockGen) { + _, blocks, _ := GenerateChainWithGenesis(gspec, engine, 1, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) // One transaction to AA, to kill it tx, _ := types.SignTx(types.NewTransaction(0, aa, @@ -3106,9 +3035,7 @@ func TestDeleteRecreateAccount(t *testing.T) { b.AddTx(tx) }) // Import the canonical chain - diskdb := rawdb.NewMemoryDatabase() - gspec.MustCommit(diskdb) - chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{ + chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, engine, vm.Config{ Debug: true, Tracer: logger.NewJSONLogger(nil, os.Stdout), }, nil, nil) @@ -3138,9 +3065,8 @@ func TestDeleteRecreateAccount(t *testing.T) { // and then the new slots exist func TestDeleteRecreateSlotsAcrossManyBlocks(t *testing.T) { var ( - // Generate a canonical chain to act as the main dataset engine = ethash.NewFaker() - db = rawdb.NewMemoryDatabase() + // A sender who makes transactions, has some funds key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") address = crypto.PubkeyToAddress(key.PublicKey) @@ -3216,7 +3142,6 @@ func TestDeleteRecreateSlotsAcrossManyBlocks(t *testing.T) { }, }, } - genesis := gspec.MustCommit(db) var nonce uint64 type expectation struct { @@ -3253,7 +3178,7 @@ func TestDeleteRecreateSlotsAcrossManyBlocks(t *testing.T) { return tx } - blocks, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 150, func(i int, b *BlockGen) { + _, blocks, _ := GenerateChainWithGenesis(gspec, engine, 150, func(i int, b *BlockGen) { var exp = new(expectation) exp.blocknum = i + 1 exp.values = make(map[int]int) @@ -3279,9 +3204,7 @@ func TestDeleteRecreateSlotsAcrossManyBlocks(t *testing.T) { current = exp }) // Import the canonical chain - diskdb := rawdb.NewMemoryDatabase() - gspec.MustCommit(diskdb) - chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{ + chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, engine, vm.Config{ //Debug: true, //Tracer: vm.NewJSONLogger(nil, os.Stdout), }, nil, nil) @@ -3324,25 +3247,23 @@ func TestDeleteRecreateSlotsAcrossManyBlocks(t *testing.T) { // TestInitThenFailCreateContract tests a pretty notorious case that happened // on mainnet over blocks 7338108, 7338110 and 7338115. -// - Block 7338108: address e771789f5cccac282f23bb7add5690e1f6ca467c is initiated -// with 0.001 ether (thus created but no code) -// - Block 7338110: a CREATE2 is attempted. The CREATE2 would deploy code on -// the same address e771789f5cccac282f23bb7add5690e1f6ca467c. However, the -// deployment fails due to OOG during initcode execution -// - Block 7338115: another tx checks the balance of -// e771789f5cccac282f23bb7add5690e1f6ca467c, and the snapshotter returned it as -// zero. +// - Block 7338108: address e771789f5cccac282f23bb7add5690e1f6ca467c is initiated +// with 0.001 ether (thus created but no code) +// - Block 7338110: a CREATE2 is attempted. The CREATE2 would deploy code on +// the same address e771789f5cccac282f23bb7add5690e1f6ca467c. However, the +// deployment fails due to OOG during initcode execution +// - Block 7338115: another tx checks the balance of +// e771789f5cccac282f23bb7add5690e1f6ca467c, and the snapshotter returned it as +// zero. // // The problem being that the snapshotter maintains a destructset, and adds items // to the destructset in case something is created "onto" an existing item. // We need to either roll back the snapDestructs, or not place it into snapDestructs // in the first place. -// func TestInitThenFailCreateContract(t *testing.T) { var ( - // Generate a canonical chain to act as the main dataset engine = ethash.NewFaker() - db = rawdb.NewMemoryDatabase() + // A sender who makes transactions, has some funds key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") address = crypto.PubkeyToAddress(key.PublicKey) @@ -3401,9 +3322,8 @@ func TestInitThenFailCreateContract(t *testing.T) { }, }, } - genesis := gspec.MustCommit(db) nonce := uint64(0) - blocks, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 4, func(i int, b *BlockGen) { + _, blocks, _ := GenerateChainWithGenesis(gspec, engine, 4, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) // One transaction to BB tx, _ := types.SignTx(types.NewTransaction(nonce, bb, @@ -3413,9 +3333,7 @@ func TestInitThenFailCreateContract(t *testing.T) { }) // Import the canonical chain - diskdb := rawdb.NewMemoryDatabase() - gspec.MustCommit(diskdb) - chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{ + chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, engine, vm.Config{ //Debug: true, //Tracer: vm.NewJSONLogger(nil, os.Stdout), }, nil, nil) @@ -3452,11 +3370,8 @@ func TestInitThenFailCreateContract(t *testing.T) { // correctly. func TestEIP2718Transition(t *testing.T) { var ( - aa = common.HexToAddress("0x000000000000000000000000000000000000aaaa") - - // Generate a canonical chain to act as the main dataset + aa = common.HexToAddress("0x000000000000000000000000000000000000aaaa") engine = ethash.NewFaker() - db = rawdb.NewMemoryDatabase() // A sender who makes transactions, has some funds key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") @@ -3479,10 +3394,9 @@ func TestEIP2718Transition(t *testing.T) { }, }, } - genesis = gspec.MustCommit(db) ) - - blocks, _ := GenerateChain(gspec.Config, genesis, engine, db, 1, func(i int, b *BlockGen) { + // Generate blocks + _, blocks, _ := GenerateChainWithGenesis(gspec, engine, 1, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) // One transaction to 0xAAAA @@ -3502,10 +3416,7 @@ func TestEIP2718Transition(t *testing.T) { }) // Import the canonical chain - diskdb := rawdb.NewMemoryDatabase() - gspec.MustCommit(diskdb) - - chain, err := NewBlockChain(diskdb, nil, gspec.Config, engine, vm.Config{}, nil, nil) + chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -3520,26 +3431,22 @@ func TestEIP2718Transition(t *testing.T) { vm.GasQuickStep*2 + params.WarmStorageReadCostEIP2929 + params.ColdSloadCostEIP2929 if block.GasUsed() != expected { t.Fatalf("incorrect amount of gas spent: expected %d, got %d", expected, block.GasUsed()) - } } // TestEIP1559Transition tests the following: // -// 1. A transaction whose gasFeeCap is greater than the baseFee is valid. -// 2. Gas accounting for access lists on EIP-1559 transactions is correct. -// 3. Only the transaction's tip will be received by the coinbase. -// 4. The transaction sender pays for both the tip and baseFee. -// 5. The coinbase receives only the partially realized tip when -// gasFeeCap - gasTipCap < baseFee. -// 6. Legacy transaction behave as expected (e.g. gasPrice = gasFeeCap = gasTipCap). +// 1. A transaction whose gasFeeCap is greater than the baseFee is valid. +// 2. Gas accounting for access lists on EIP-1559 transactions is correct. +// 3. Only the transaction's tip will be received by the coinbase. +// 4. The transaction sender pays for both the tip and baseFee. +// 5. The coinbase receives only the partially realized tip when +// gasFeeCap - gasTipCap < baseFee. +// 6. Legacy transaction behave as expected (e.g. gasPrice = gasFeeCap = gasTipCap). func TestEIP1559Transition(t *testing.T) { var ( - aa = common.HexToAddress("0x000000000000000000000000000000000000aaaa") - - // Generate a canonical chain to act as the main dataset + aa = common.HexToAddress("0x000000000000000000000000000000000000aaaa") engine = ethash.NewFaker() - db = rawdb.NewMemoryDatabase() // A sender who makes transactions, has some funds key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") @@ -3569,10 +3476,9 @@ func TestEIP1559Transition(t *testing.T) { gspec.Config.BerlinBlock = common.Big0 gspec.Config.LondonBlock = common.Big0 - genesis := gspec.MustCommit(db) signer := types.LatestSigner(gspec.Config) - blocks, _ := GenerateChain(gspec.Config, genesis, engine, db, 1, func(i int, b *BlockGen) { + genDb, blocks, _ := GenerateChainWithGenesis(gspec, engine, 1, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) // One transaction to 0xAAAA @@ -3596,11 +3502,7 @@ func TestEIP1559Transition(t *testing.T) { b.AddTx(tx) }) - - diskdb := rawdb.NewMemoryDatabase() - gspec.MustCommit(diskdb) - - chain, err := NewBlockChain(diskdb, nil, gspec.Config, engine, vm.Config{}, nil, nil) + chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -3636,7 +3538,7 @@ func TestEIP1559Transition(t *testing.T) { t.Fatalf("sender balance incorrect: expected %d, got %d", expected, actual) } - blocks, _ = GenerateChain(gspec.Config, block, engine, db, 1, func(i int, b *BlockGen) { + blocks, _ = GenerateChain(gspec.Config, block, engine, genDb, 1, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{2}) txdata := &types.LegacyTx{ @@ -3683,7 +3585,6 @@ func TestSetCanonical(t *testing.T) { //log.Root().SetHandler(log.LvlFilterHandler(log.LvlDebug, log.StreamHandler(os.Stderr, log.TerminalFormat(true)))) var ( - db = rawdb.NewMemoryDatabase() key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") address = crypto.PubkeyToAddress(key.PublicKey) funds = big.NewInt(100000000000000000) @@ -3692,22 +3593,18 @@ func TestSetCanonical(t *testing.T) { Alloc: GenesisAlloc{address: {Balance: funds}}, BaseFee: big.NewInt(params.InitialBaseFee), } - genesis = gspec.MustCommit(db) - signer = types.LatestSigner(gspec.Config) - engine = ethash.NewFaker() + signer = types.LatestSigner(gspec.Config) + engine = ethash.NewFaker() ) // Generate and import the canonical chain - canon, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 2*TriesInMemory, func(i int, gen *BlockGen) { + _, canon, _ := GenerateChainWithGenesis(gspec, engine, 2*TriesInMemory, func(i int, gen *BlockGen) { tx, err := types.SignTx(types.NewTransaction(gen.TxNonce(address), common.Address{0x00}, big.NewInt(1000), params.TxGas, gen.header.BaseFee, nil), signer, key) if err != nil { panic(err) } gen.AddTx(tx) }) - diskdb := rawdb.NewMemoryDatabase() - gspec.MustCommit(diskdb) - - chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil, nil) + chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -3716,7 +3613,7 @@ func TestSetCanonical(t *testing.T) { } // Generate the side chain and import them - side, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 2*TriesInMemory, func(i int, gen *BlockGen) { + _, side, _ := GenerateChainWithGenesis(gspec, engine, 2*TriesInMemory, func(i int, gen *BlockGen) { tx, err := types.SignTx(types.NewTransaction(gen.TxNonce(address), common.Address{0x00}, big.NewInt(1), params.TxGas, gen.header.BaseFee, nil), signer, key) if err != nil { panic(err) @@ -3798,22 +3695,18 @@ func TestCanonicalHashMarker(t *testing.T) { } for _, c := range cases { var ( - db = rawdb.NewMemoryDatabase() gspec = &Genesis{ Config: params.TestChainConfig, Alloc: GenesisAlloc{}, BaseFee: big.NewInt(params.InitialBaseFee), } - genesis = gspec.MustCommit(db) - engine = ethash.NewFaker() + engine = ethash.NewFaker() ) - forkA, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, c.forkA, func(i int, gen *BlockGen) {}) - forkB, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, c.forkB, func(i int, gen *BlockGen) {}) + _, forkA, _ := GenerateChainWithGenesis(gspec, engine, c.forkA, func(i int, gen *BlockGen) {}) + _, forkB, _ := GenerateChainWithGenesis(gspec, engine, c.forkB, func(i int, gen *BlockGen) {}) // Initialize test chain - diskdb := rawdb.NewMemoryDatabase() - gspec.MustCommit(diskdb) - chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil, nil) + chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -3867,3 +3760,208 @@ func TestCanonicalHashMarker(t *testing.T) { } } } + +// TestTxIndexer tests the tx indexes are updated correctly. +func TestTxIndexer(t *testing.T) { + var ( + testBankKey, _ = crypto.GenerateKey() + testBankAddress = crypto.PubkeyToAddress(testBankKey.PublicKey) + testBankFunds = big.NewInt(1000000000000000000) + + gspec = &Genesis{ + Config: params.TestChainConfig, + Alloc: GenesisAlloc{testBankAddress: {Balance: testBankFunds}}, + BaseFee: big.NewInt(params.InitialBaseFee), + } + engine = ethash.NewFaker() + nonce = uint64(0) + ) + _, blocks, receipts := GenerateChainWithGenesis(gspec, engine, 128, func(i int, gen *BlockGen) { + tx, _ := types.SignTx(types.NewTransaction(nonce, common.HexToAddress("0xdeadbeef"), big.NewInt(1000), params.TxGas, big.NewInt(10*params.InitialBaseFee), nil), types.HomesteadSigner{}, testBankKey) + gen.AddTx(tx) + nonce += 1 + }) + + // verifyIndexes checks if the transaction indexes are present or not + // of the specified block. + verifyIndexes := func(db ethdb.Database, number uint64, exist bool) { + if number == 0 { + return + } + block := blocks[number-1] + for _, tx := range block.Transactions() { + lookup := rawdb.ReadTxLookupEntry(db, tx.Hash()) + if exist && lookup == nil { + t.Fatalf("missing %d %x", number, tx.Hash().Hex()) + } + if !exist && lookup != nil { + t.Fatalf("unexpected %d %x", number, tx.Hash().Hex()) + } + } + } + // verifyRange runs verifyIndexes for a range of blocks, from and to are included. + verifyRange := func(db ethdb.Database, from, to uint64, exist bool) { + for number := from; number <= to; number += 1 { + verifyIndexes(db, number, exist) + } + } + verify := func(db ethdb.Database, expTail uint64) { + tail := rawdb.ReadTxIndexTail(db) + if tail == nil { + t.Fatal("Failed to write tx index tail") + } + if *tail != expTail { + t.Fatalf("Unexpected tx index tail, want %v, got %d", expTail, *tail) + } + if *tail != 0 { + verifyRange(db, 0, *tail-1, false) + } + verifyRange(db, *tail, 128, true) + } + + var cases = []struct { + limitA uint64 + tailA uint64 + limitB uint64 + tailB uint64 + limitC uint64 + tailC uint64 + }{ + { + // LimitA: 0 + // TailA: 0 + // + // all blocks are indexed + limitA: 0, + tailA: 0, + + // LimitB: 1 + // TailB: 128 + // + // block-128 is indexed + limitB: 1, + tailB: 128, + + // LimitB: 64 + // TailB: 65 + // + // block [65, 128] are indexed + limitC: 64, + tailC: 65, + }, + { + // LimitA: 64 + // TailA: 65 + // + // block [65, 128] are indexed + limitA: 64, + tailA: 65, + + // LimitB: 1 + // TailB: 128 + // + // block-128 is indexed + limitB: 1, + tailB: 128, + + // LimitB: 64 + // TailB: 65 + // + // block [65, 128] are indexed + limitC: 64, + tailC: 65, + }, + { + // LimitA: 127 + // TailA: 2 + // + // block [2, 128] are indexed + limitA: 127, + tailA: 2, + + // LimitB: 1 + // TailB: 128 + // + // block-128 is indexed + limitB: 1, + tailB: 128, + + // LimitB: 64 + // TailB: 65 + // + // block [65, 128] are indexed + limitC: 64, + tailC: 65, + }, + { + // LimitA: 128 + // TailA: 1 + // + // block [2, 128] are indexed + limitA: 128, + tailA: 1, + + // LimitB: 1 + // TailB: 128 + // + // block-128 is indexed + limitB: 1, + tailB: 128, + + // LimitB: 64 + // TailB: 65 + // + // block [65, 128] are indexed + limitC: 64, + tailC: 65, + }, + { + // LimitA: 129 + // TailA: 0 + // + // block [0, 128] are indexed + limitA: 129, + tailA: 0, + + // LimitB: 1 + // TailB: 128 + // + // block-128 is indexed + limitB: 1, + tailB: 128, + + // LimitB: 64 + // TailB: 65 + // + // block [65, 128] are indexed + limitC: 64, + tailC: 65, + }, + } + for _, c := range cases { + frdir := t.TempDir() + db, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), frdir, "", false) + rawdb.WriteAncientBlocks(db, append([]*types.Block{gspec.ToBlock()}, blocks...), append([]types.Receipts{{}}, receipts...), big.NewInt(0)) + + // Index the initial blocks from ancient store + chain, _ := NewBlockChain(db, nil, gspec, nil, engine, vm.Config{}, nil, &c.limitA) + chain.indexBlocks(nil, 128, make(chan struct{})) + verify(db, c.tailA) + + chain.SetTxLookupLimit(c.limitB) + chain.indexBlocks(rawdb.ReadTxIndexTail(db), 128, make(chan struct{})) + verify(db, c.tailB) + + chain.SetTxLookupLimit(c.limitC) + chain.indexBlocks(rawdb.ReadTxIndexTail(db), 128, make(chan struct{})) + verify(db, c.tailC) + + // Recover all indexes + chain.SetTxLookupLimit(0) + chain.indexBlocks(rawdb.ReadTxIndexTail(db), 128, make(chan struct{})) + verify(db, 0) + + db.Close() + os.RemoveAll(frdir) + } +} diff --git a/core/bloom_indexer.go b/core/bloom_indexer.go index 856746a1c0883..68a35d811e41e 100644 --- a/core/bloom_indexer.go +++ b/core/bloom_indexer.go @@ -75,7 +75,7 @@ func (b *BloomIndexer) Process(ctx context.Context, header *types.Header) error // Commit implements core.ChainIndexerBackend, finalizing the bloom section and // writing it out into the database. func (b *BloomIndexer) Commit() error { - batch := b.db.NewBatch() + batch := b.db.NewBatchWithSize((int(b.size) / 8) * types.BloomBitLength) for i := 0; i < types.BloomBitLength; i++ { bits, err := b.gen.Bitset(uint(i)) if err != nil { diff --git a/core/bloombits/matcher_test.go b/core/bloombits/matcher_test.go index 923579221f516..93d4632b8587b 100644 --- a/core/bloombits/matcher_test.go +++ b/core/bloombits/matcher_test.go @@ -124,13 +124,13 @@ func makeRandomIndexes(lengths []int, max int) [][]bloomIndexes { // testMatcherDiffBatches runs the given matches test in single-delivery and also // in batches delivery mode, verifying that all kinds of deliveries are handled -// correctly withn. +// correctly within. func testMatcherDiffBatches(t *testing.T, filter [][]bloomIndexes, start, blocks uint64, intermittent bool, retrievals uint32) { singleton := testMatcher(t, filter, start, blocks, intermittent, retrievals, 1) batched := testMatcher(t, filter, start, blocks, intermittent, retrievals, 16) if singleton != batched { - t.Errorf("filter = %v blocks = %v intermittent = %v: request count mismatch, %v in signleton vs. %v in batched mode", filter, blocks, intermittent, singleton, batched) + t.Errorf("filter = %v blocks = %v intermittent = %v: request count mismatch, %v in singleton vs. %v in batched mode", filter, blocks, intermittent, singleton, batched) } } diff --git a/core/chain_makers.go b/core/chain_makers.go index c7bf60a4b06ee..88a1c4e870242 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -23,6 +23,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/consensus/misc" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" @@ -284,6 +285,19 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse return blocks, receipts } +// GenerateChainWithGenesis is a wrapper of GenerateChain which will initialize +// genesis block to database first according to the provided genesis specification +// then generate chain on top. +func GenerateChainWithGenesis(genesis *Genesis, engine consensus.Engine, n int, gen func(int, *BlockGen)) (ethdb.Database, []*types.Block, []types.Receipts) { + db := rawdb.NewMemoryDatabase() + _, err := genesis.Commit(db) + if err != nil { + panic(err) + } + blocks, receipts := GenerateChain(genesis.Config, genesis.ToBlock(), engine, db, n, gen) + return db, blocks, receipts +} + func makeHeader(chain consensus.ChainReader, parent *types.Block, state *state.StateDB, engine consensus.Engine) *types.Header { var time uint64 if parent.Time() == 0 { @@ -316,8 +330,8 @@ func makeHeader(chain consensus.ChainReader, parent *types.Block, state *state.S } // makeHeaderChain creates a deterministic chain of headers rooted at parent. -func makeHeaderChain(parent *types.Header, n int, engine consensus.Engine, db ethdb.Database, seed int) []*types.Header { - blocks := makeBlockChain(types.NewBlockWithHeader(parent), n, engine, db, seed) +func makeHeaderChain(chainConfig *params.ChainConfig, parent *types.Header, n int, engine consensus.Engine, db ethdb.Database, seed int) []*types.Header { + blocks := makeBlockChain(chainConfig, types.NewBlockWithHeader(parent), n, engine, db, seed) headers := make([]*types.Header, len(blocks)) for i, block := range blocks { headers[i] = block.Header() @@ -325,14 +339,32 @@ func makeHeaderChain(parent *types.Header, n int, engine consensus.Engine, db et return headers } +// makeHeaderChainWithGenesis creates a deterministic chain of headers from genesis. +func makeHeaderChainWithGenesis(genesis *Genesis, n int, engine consensus.Engine, seed int) (ethdb.Database, []*types.Header) { + db, blocks := makeBlockChainWithGenesis(genesis, n, engine, seed) + headers := make([]*types.Header, len(blocks)) + for i, block := range blocks { + headers[i] = block.Header() + } + return db, headers +} + // makeBlockChain creates a deterministic chain of blocks rooted at parent. -func makeBlockChain(parent *types.Block, n int, engine consensus.Engine, db ethdb.Database, seed int) []*types.Block { - blocks, _ := GenerateChain(params.TestChainConfig, parent, engine, db, n, func(i int, b *BlockGen) { +func makeBlockChain(chainConfig *params.ChainConfig, parent *types.Block, n int, engine consensus.Engine, db ethdb.Database, seed int) []*types.Block { + blocks, _ := GenerateChain(chainConfig, parent, engine, db, n, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{0: byte(seed), 19: byte(i)}) }) return blocks } +// makeBlockChain creates a deterministic chain of blocks from genesis +func makeBlockChainWithGenesis(genesis *Genesis, n int, engine consensus.Engine, seed int) (ethdb.Database, []*types.Block) { + db, blocks, _ := GenerateChainWithGenesis(genesis, engine, n, func(i int, b *BlockGen) { + b.SetCoinbase(common.Address{0: byte(seed), 19: byte(i)}) + }) + return db, blocks +} + type fakeChainReader struct { config *params.ChainConfig } diff --git a/core/chain_makers_test.go b/core/chain_makers_test.go index 85a029f7c7572..166ac3f227fc9 100644 --- a/core/chain_makers_test.go +++ b/core/chain_makers_test.go @@ -79,7 +79,7 @@ func ExampleGenerateChain() { }) // Import the chain. This runs all block validation rules. - blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) + blockchain, _ := NewBlockChain(db, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) defer blockchain.Stop() if i, err := blockchain.InsertChain(chain); err != nil { diff --git a/core/dao_test.go b/core/dao_test.go index c9c765a3832a3..44405447deaa2 100644 --- a/core/dao_test.go +++ b/core/dao_test.go @@ -30,32 +30,39 @@ import ( // blocks based on their extradata fields. func TestDAOForkRangeExtradata(t *testing.T) { forkBlock := big.NewInt(32) + chainConfig := *params.NonActivatedConfig + chainConfig.HomesteadBlock = big.NewInt(0) // Generate a common prefix for both pro-forkers and non-forkers - db := rawdb.NewMemoryDatabase() - gspec := &Genesis{BaseFee: big.NewInt(params.InitialBaseFee)} - genesis := gspec.MustCommit(db) - prefix, _ := GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), db, int(forkBlock.Int64()-1), func(i int, gen *BlockGen) {}) + gspec := &Genesis{ + BaseFee: big.NewInt(params.InitialBaseFee), + Config: &chainConfig, + } + genDb, prefix, _ := GenerateChainWithGenesis(gspec, ethash.NewFaker(), int(forkBlock.Int64()-1), func(i int, gen *BlockGen) {}) // Create the concurrent, conflicting two nodes proDb := rawdb.NewMemoryDatabase() - gspec.MustCommit(proDb) - - proConf := *params.TestChainConfig + proConf := *params.NonActivatedConfig + proConf.HomesteadBlock = big.NewInt(0) proConf.DAOForkBlock = forkBlock proConf.DAOForkSupport = true - - proBc, _ := NewBlockChain(proDb, nil, &proConf, ethash.NewFaker(), vm.Config{}, nil, nil) + progspec := &Genesis{ + BaseFee: big.NewInt(params.InitialBaseFee), + Config: &proConf, + } + proBc, _ := NewBlockChain(proDb, nil, progspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) defer proBc.Stop() conDb := rawdb.NewMemoryDatabase() - gspec.MustCommit(conDb) - - conConf := *params.TestChainConfig + conConf := *params.NonActivatedConfig + conConf.HomesteadBlock = big.NewInt(0) conConf.DAOForkBlock = forkBlock conConf.DAOForkSupport = false - - conBc, _ := NewBlockChain(conDb, nil, &conConf, ethash.NewFaker(), vm.Config{}, nil, nil) + congspec := &Genesis{ + BaseFee: big.NewInt(params.InitialBaseFee), + Config: &conConf, + } + conBc, _ := NewBlockChain(conDb, nil, congspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) defer conBc.Stop() if _, err := proBc.InsertChain(prefix); err != nil { @@ -67,9 +74,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { // Try to expand both pro-fork and non-fork chains iteratively with other camp's blocks for i := int64(0); i < params.DAOForkExtraRange.Int64(); i++ { // Create a pro-fork block, and try to feed into the no-fork chain - db = rawdb.NewMemoryDatabase() - gspec.MustCommit(db) - bc, _ := NewBlockChain(db, nil, &conConf, ethash.NewFaker(), vm.Config{}, nil, nil) + bc, _ := NewBlockChain(rawdb.NewMemoryDatabase(), nil, congspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) defer bc.Stop() blocks := conBc.GetBlocksFromHash(conBc.CurrentBlock().Hash(), int(conBc.CurrentBlock().NumberU64())) @@ -82,19 +87,17 @@ func TestDAOForkRangeExtradata(t *testing.T) { if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, true, nil); err != nil { t.Fatalf("failed to commit contra-fork head for expansion: %v", err) } - blocks, _ = GenerateChain(&proConf, conBc.CurrentBlock(), ethash.NewFaker(), db, 1, func(i int, gen *BlockGen) {}) + blocks, _ = GenerateChain(&proConf, conBc.CurrentBlock(), ethash.NewFaker(), genDb, 1, func(i int, gen *BlockGen) {}) if _, err := conBc.InsertChain(blocks); err == nil { t.Fatalf("contra-fork chain accepted pro-fork block: %v", blocks[0]) } // Create a proper no-fork block for the contra-forker - blocks, _ = GenerateChain(&conConf, conBc.CurrentBlock(), ethash.NewFaker(), db, 1, func(i int, gen *BlockGen) {}) + blocks, _ = GenerateChain(&conConf, conBc.CurrentBlock(), ethash.NewFaker(), genDb, 1, func(i int, gen *BlockGen) {}) if _, err := conBc.InsertChain(blocks); err != nil { t.Fatalf("contra-fork chain didn't accepted no-fork block: %v", err) } // Create a no-fork block, and try to feed into the pro-fork chain - db = rawdb.NewMemoryDatabase() - gspec.MustCommit(db) - bc, _ = NewBlockChain(db, nil, &proConf, ethash.NewFaker(), vm.Config{}, nil, nil) + bc, _ = NewBlockChain(rawdb.NewMemoryDatabase(), nil, progspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) defer bc.Stop() blocks = proBc.GetBlocksFromHash(proBc.CurrentBlock().Hash(), int(proBc.CurrentBlock().NumberU64())) @@ -107,20 +110,18 @@ func TestDAOForkRangeExtradata(t *testing.T) { if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, true, nil); err != nil { t.Fatalf("failed to commit pro-fork head for expansion: %v", err) } - blocks, _ = GenerateChain(&conConf, proBc.CurrentBlock(), ethash.NewFaker(), db, 1, func(i int, gen *BlockGen) {}) + blocks, _ = GenerateChain(&conConf, proBc.CurrentBlock(), ethash.NewFaker(), genDb, 1, func(i int, gen *BlockGen) {}) if _, err := proBc.InsertChain(blocks); err == nil { t.Fatalf("pro-fork chain accepted contra-fork block: %v", blocks[0]) } // Create a proper pro-fork block for the pro-forker - blocks, _ = GenerateChain(&proConf, proBc.CurrentBlock(), ethash.NewFaker(), db, 1, func(i int, gen *BlockGen) {}) + blocks, _ = GenerateChain(&proConf, proBc.CurrentBlock(), ethash.NewFaker(), genDb, 1, func(i int, gen *BlockGen) {}) if _, err := proBc.InsertChain(blocks); err != nil { t.Fatalf("pro-fork chain didn't accepted pro-fork block: %v", err) } } // Verify that contra-forkers accept pro-fork extra-datas after forking finishes - db = rawdb.NewMemoryDatabase() - gspec.MustCommit(db) - bc, _ := NewBlockChain(db, nil, &conConf, ethash.NewFaker(), vm.Config{}, nil, nil) + bc, _ := NewBlockChain(rawdb.NewMemoryDatabase(), nil, congspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) defer bc.Stop() blocks := conBc.GetBlocksFromHash(conBc.CurrentBlock().Hash(), int(conBc.CurrentBlock().NumberU64())) @@ -133,14 +134,12 @@ func TestDAOForkRangeExtradata(t *testing.T) { if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, true, nil); err != nil { t.Fatalf("failed to commit contra-fork head for expansion: %v", err) } - blocks, _ = GenerateChain(&proConf, conBc.CurrentBlock(), ethash.NewFaker(), db, 1, func(i int, gen *BlockGen) {}) + blocks, _ = GenerateChain(&proConf, conBc.CurrentBlock(), ethash.NewFaker(), genDb, 1, func(i int, gen *BlockGen) {}) if _, err := conBc.InsertChain(blocks); err != nil { t.Fatalf("contra-fork chain didn't accept pro-fork block post-fork: %v", err) } // Verify that pro-forkers accept contra-fork extra-datas after forking finishes - db = rawdb.NewMemoryDatabase() - gspec.MustCommit(db) - bc, _ = NewBlockChain(db, nil, &proConf, ethash.NewFaker(), vm.Config{}, nil, nil) + bc, _ = NewBlockChain(rawdb.NewMemoryDatabase(), nil, progspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) defer bc.Stop() blocks = proBc.GetBlocksFromHash(proBc.CurrentBlock().Hash(), int(proBc.CurrentBlock().NumberU64())) @@ -153,7 +152,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, true, nil); err != nil { t.Fatalf("failed to commit pro-fork head for expansion: %v", err) } - blocks, _ = GenerateChain(&conConf, proBc.CurrentBlock(), ethash.NewFaker(), db, 1, func(i int, gen *BlockGen) {}) + blocks, _ = GenerateChain(&conConf, proBc.CurrentBlock(), ethash.NewFaker(), genDb, 1, func(i int, gen *BlockGen) {}) if _, err := proBc.InsertChain(blocks); err != nil { t.Fatalf("pro-fork chain didn't accept contra-fork block post-fork: %v", err) } diff --git a/core/error.go b/core/error.go index 51ebefc137bcb..5b69c8dcaf26a 100644 --- a/core/error.go +++ b/core/error.go @@ -91,7 +91,7 @@ var ( ErrFeeCapVeryHigh = errors.New("max fee per gas higher than 2^256-1") // ErrFeeCapTooLow is returned if the transaction fee cap is less than the - // the base fee of the block. + // base fee of the block. ErrFeeCapTooLow = errors.New("max fee per gas less than block base fee") // ErrSenderNoEOA is returned if the sender of a transaction is a contract. diff --git a/core/evm.go b/core/evm.go index 21e2639a5f63d..e929da25eaeea 100644 --- a/core/evm.go +++ b/core/evm.go @@ -31,7 +31,7 @@ type ChainContext interface { // Engine retrieves the chain's consensus engine. Engine() consensus.Engine - // GetHeader returns the hash corresponding to their hash. + // GetHeader returns the header corresponding to the hash/number argument pair. GetHeader(common.Hash, uint64) *types.Header } diff --git a/core/forkid/forkid_test.go b/core/forkid/forkid_test.go index b0ee59b9eb7b2..2a0fb167d5166 100644 --- a/core/forkid/forkid_test.go +++ b/core/forkid/forkid_test.go @@ -31,7 +31,7 @@ import ( // the correct fork ID. func TestCreation(t *testing.T) { mergeConfig := *params.MainnetChainConfig - mergeConfig.MergeForkBlock = big.NewInt(15000000) + mergeConfig.MergeNetsplitBlock = big.NewInt(18000000) type testcase struct { head uint64 want ID @@ -68,8 +68,10 @@ func TestCreation(t *testing.T) { {12964999, ID{Hash: checksumToBytes(0x0eb440f6), Next: 12965000}}, // Last Berlin block {12965000, ID{Hash: checksumToBytes(0xb715077d), Next: 13773000}}, // First London block {13772999, ID{Hash: checksumToBytes(0xb715077d), Next: 13773000}}, // Last London block - {13773000, ID{Hash: checksumToBytes(0x20c327fc), Next: 0}}, // First Arrow Glacier block - {20000000, ID{Hash: checksumToBytes(0x20c327fc), Next: 0}}, // Future Arrow Glacier block + {13773000, ID{Hash: checksumToBytes(0x20c327fc), Next: 15050000}}, // First Arrow Glacier block + {15049999, ID{Hash: checksumToBytes(0x20c327fc), Next: 15050000}}, // Last Arrow Glacier block + {15050000, ID{Hash: checksumToBytes(0xf0afd0e3), Next: 0}}, // First Gray Glacier block + {20000000, ID{Hash: checksumToBytes(0xf0afd0e3), Next: 0}}, // Future Gray Glacier block }, }, // Ropsten test cases @@ -136,6 +138,16 @@ func TestCreation(t *testing.T) { {6000000, ID{Hash: checksumToBytes(0xB8C6299D), Next: 0}}, // Future London block }, }, + // Sepolia test cases + { + params.SepoliaChainConfig, + params.SepoliaGenesisHash, + []testcase{ + {0, ID{Hash: checksumToBytes(0xfe3366e7), Next: 1735371}}, // Unsynced, last Frontier, Homestead, Tangerine, Spurious, Byzantium, Constantinople, Petersburg, Istanbul, Berlin and first London block + {1735370, ID{Hash: checksumToBytes(0xfe3366e7), Next: 1735371}}, // Last London block + {1735371, ID{Hash: checksumToBytes(0xb96cbd13), Next: 0}}, // First MergeNetsplit block + }, + }, // Merge test cases { &mergeConfig, @@ -163,9 +175,11 @@ func TestCreation(t *testing.T) { {12964999, ID{Hash: checksumToBytes(0x0eb440f6), Next: 12965000}}, // Last Berlin block {12965000, ID{Hash: checksumToBytes(0xb715077d), Next: 13773000}}, // First London block {13772999, ID{Hash: checksumToBytes(0xb715077d), Next: 13773000}}, // Last London block - {13773000, ID{Hash: checksumToBytes(0x20c327fc), Next: 15000000}}, // First Arrow Glacier block - {15000000, ID{Hash: checksumToBytes(0xe3abe201), Next: 0}}, // First Merge Start block - {20000000, ID{Hash: checksumToBytes(0xe3abe201), Next: 0}}, // Future Merge Start block + {13773000, ID{Hash: checksumToBytes(0x20c327fc), Next: 15050000}}, // First Arrow Glacier block + {15049999, ID{Hash: checksumToBytes(0x20c327fc), Next: 15050000}}, // Last Arrow Glacier block + {15050000, ID{Hash: checksumToBytes(0xf0afd0e3), Next: 18000000}}, // First Gray Glacier block + {18000000, ID{Hash: checksumToBytes(0x4fb8a872), Next: 0}}, // First Merge Start block + {20000000, ID{Hash: checksumToBytes(0x4fb8a872), Next: 0}}, // Future Merge Start block }, }, } @@ -242,11 +256,11 @@ func TestValidation(t *testing.T) { // Local is mainnet Petersburg, remote is Rinkeby Petersburg. {7987396, ID{Hash: checksumToBytes(0xafec6b27), Next: 0}, ErrLocalIncompatibleOrStale}, - // Local is mainnet Arrow Glacier, far in the future. Remote announces Gopherium (non existing fork) + // Local is mainnet Gray Glacier, far in the future. Remote announces Gopherium (non existing fork) // at some future block 88888888, for itself, but past block for local. Local is incompatible. // // This case detects non-upgraded nodes with majority hash power (typical Ropsten mess). - {88888888, ID{Hash: checksumToBytes(0x20c327fc), Next: 88888888}, ErrLocalIncompatibleOrStale}, + {88888888, ID{Hash: checksumToBytes(0xf0afd0e3), Next: 88888888}, ErrLocalIncompatibleOrStale}, // Local is mainnet Byzantium. Remote is also in Byzantium, but announces Gopherium (non existing // fork) at block 7279999, before Petersburg. Local is incompatible. diff --git a/core/genesis.go b/core/genesis.go index 64ee99c5443de..8b855147972db 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -65,6 +65,41 @@ type Genesis struct { BaseFee *big.Int `json:"baseFeePerGas"` } +func ReadGenesis(db ethdb.Database) (*Genesis, error) { + var genesis Genesis + stored := rawdb.ReadCanonicalHash(db, 0) + if (stored == common.Hash{}) { + return nil, fmt.Errorf("invalid genesis hash in database: %x", stored) + } + blob := rawdb.ReadGenesisStateSpec(db, stored) + if blob == nil { + return nil, fmt.Errorf("genesis state missing from db") + } + if len(blob) != 0 { + if err := genesis.Alloc.UnmarshalJSON(blob); err != nil { + return nil, fmt.Errorf("could not unmarshal genesis state json: %s", err) + } + } + genesis.Config = rawdb.ReadChainConfig(db, stored) + if genesis.Config == nil { + return nil, fmt.Errorf("genesis config missing from db") + } + genesisBlock := rawdb.ReadBlock(db, stored, 0) + if genesisBlock == nil { + return nil, fmt.Errorf("genesis block missing from db") + } + genesisHeader := genesisBlock.Header() + genesis.Nonce = genesisHeader.Nonce.Uint64() + genesis.Timestamp = genesisHeader.Time + genesis.ExtraData = genesisHeader.Extra + genesis.GasLimit = genesisHeader.GasLimit + genesis.Difficulty = genesisHeader.Difficulty + genesis.Mixhash = genesisHeader.MixDigest + genesis.Coinbase = genesisHeader.Coinbase + + return &genesis, nil +} + // GenesisAlloc specifies the initial state that is part of the genesis block. type GenesisAlloc map[common.Address]GenesisAccount @@ -80,10 +115,12 @@ func (ga *GenesisAlloc) UnmarshalJSON(data []byte) error { return nil } -// flush adds allocated genesis accounts into a fresh new statedb and -// commit the state changes into the given database handler. -func (ga *GenesisAlloc) flush(db ethdb.Database) (common.Hash, error) { - statedb, err := state.New(common.Hash{}, state.NewDatabase(db), nil) +// deriveHash computes the state root according to the genesis specification. +func (ga *GenesisAlloc) deriveHash() (common.Hash, error) { + // Create an ephemeral in-memory database for computing hash, + // all the derived states will be discarded to not pollute disk. + db := state.NewDatabase(rawdb.NewMemoryDatabase()) + statedb, err := state.New(common.Hash{}, db, nil) if err != nil { return common.Hash{}, err } @@ -95,25 +132,39 @@ func (ga *GenesisAlloc) flush(db ethdb.Database) (common.Hash, error) { statedb.SetState(addr, key, value) } } + return statedb.Commit(false) +} + +// flush is very similar with deriveHash, but the main difference is +// all the generated states will be persisted into the given database. +// Also, the genesis state specification will be flushed as well. +func (ga *GenesisAlloc) flush(db ethdb.Database) error { + statedb, err := state.New(common.Hash{}, state.NewDatabaseWithConfig(db, &trie.Config{Preimages: true}), nil) + if err != nil { + return err + } + for addr, account := range *ga { + statedb.AddBalance(addr, account.Balance) + statedb.SetCode(addr, account.Code) + statedb.SetNonce(addr, account.Nonce) + for key, value := range account.Storage { + statedb.SetState(addr, key, value) + } + } root, err := statedb.Commit(false) if err != nil { - return common.Hash{}, err + return err } err = statedb.Database().TrieDB().Commit(root, true, nil) if err != nil { - return common.Hash{}, err + return err } - return root, nil -} - -// write writes the json marshaled genesis state into database -// with the given block hash as the unique identifier. -func (ga *GenesisAlloc) write(db ethdb.KeyValueWriter, hash common.Hash) error { + // Marshal the genesis state specification and persist. blob, err := json.Marshal(ga) if err != nil { return err } - rawdb.WriteGenesisState(db, hash, blob) + rawdb.WriteGenesisStateSpec(db, root, blob) return nil } @@ -121,7 +172,7 @@ func (ga *GenesisAlloc) write(db ethdb.KeyValueWriter, hash common.Hash) error { // hash and commits them into the given database handler. func CommitGenesisState(db ethdb.Database, hash common.Hash) error { var alloc GenesisAlloc - blob := rawdb.ReadGenesisState(db, hash) + blob := rawdb.ReadGenesisStateSpec(db, hash) if len(blob) != 0 { if err := alloc.UnmarshalJSON(blob); err != nil { return err @@ -151,8 +202,7 @@ func CommitGenesisState(db ethdb.Database, hash common.Hash) error { return errors.New("not found") } } - _, err := alloc.flush(db) - return err + return alloc.flush(db) } // GenesisAccount is an account in the state of the genesis block. @@ -196,7 +246,6 @@ func (h *storageJSON) UnmarshalText(text []byte) error { } offset := len(h) - len(text)/2 // pad on the left if _, err := hex.Decode(h[offset:], text); err != nil { - fmt.Println(err) return fmt.Errorf("invalid hex storage key/value %q", text) } return nil @@ -216,13 +265,19 @@ func (e *GenesisMismatchError) Error() string { return fmt.Sprintf("database contains incompatible genesis (have %x, new %x)", e.Stored, e.New) } +// ChainOverrides contains the changes to chain config. +type ChainOverrides struct { + OverrideTerminalTotalDifficulty *big.Int + OverrideTerminalTotalDifficultyPassed *bool +} + // SetupGenesisBlock writes or updates the genesis block in db. // The block that will be used is: // -// genesis == nil genesis != nil -// +------------------------------------------ -// db has no genesis | main-net default | genesis -// db has genesis | from DB | genesis (if compatible) +// genesis == nil genesis != nil +// +------------------------------------------ +// db has no genesis | main-net default | genesis +// db has genesis | from DB | genesis (if compatible) // // The stored chain configuration will be updated if it is compatible (i.e. does not // specify a fork block below the local head block). In case of a conflict, the @@ -230,13 +285,25 @@ func (e *GenesisMismatchError) Error() string { // // The returned chain configuration is never nil. func SetupGenesisBlock(db ethdb.Database, genesis *Genesis) (*params.ChainConfig, common.Hash, error) { - return SetupGenesisBlockWithOverride(db, genesis, nil, nil) + return SetupGenesisBlockWithOverride(db, genesis, nil) } -func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, overrideArrowGlacier, overrideTerminalTotalDifficulty *big.Int) (*params.ChainConfig, common.Hash, error) { +func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, overrides *ChainOverrides) (*params.ChainConfig, common.Hash, error) { if genesis != nil && genesis.Config == nil { return params.AllEthashProtocolChanges, common.Hash{}, errGenesisNoConfig } + + applyOverrides := func(config *params.ChainConfig) { + if config != nil { + if overrides != nil && overrides.OverrideTerminalTotalDifficulty != nil { + config.TerminalTotalDifficulty = overrides.OverrideTerminalTotalDifficulty + } + if overrides != nil && overrides.OverrideTerminalTotalDifficultyPassed != nil { + config.TerminalTotalDifficultyPassed = *overrides.OverrideTerminalTotalDifficultyPassed + } + } + } + // Just commit the new block if there is no stored genesis block. stored := rawdb.ReadCanonicalHash(db, 0) if (stored == common.Hash{}) { @@ -250,6 +317,7 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, override if err != nil { return genesis.Config, common.Hash{}, err } + applyOverrides(genesis.Config) return genesis.Config, block.Hash(), nil } // We have the genesis block in database(perhaps in ancient database) @@ -260,7 +328,7 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, override genesis = DefaultGenesisBlock() } // Ensure the stored genesis matches with the given one. - hash := genesis.ToBlock(nil).Hash() + hash := genesis.ToBlock().Hash() if hash != stored { return genesis.Config, hash, &GenesisMismatchError{stored, hash} } @@ -268,23 +336,19 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, override if err != nil { return genesis.Config, hash, err } + applyOverrides(genesis.Config) return genesis.Config, block.Hash(), nil } // Check whether the genesis block is already written. if genesis != nil { - hash := genesis.ToBlock(nil).Hash() + hash := genesis.ToBlock().Hash() if hash != stored { return genesis.Config, hash, &GenesisMismatchError{stored, hash} } } // Get the existing chain configuration. newcfg := genesis.configOrDefault(stored) - if overrideArrowGlacier != nil { - newcfg.ArrowGlacierBlock = overrideArrowGlacier - } - if overrideTerminalTotalDifficulty != nil { - newcfg.TerminalTotalDifficulty = overrideTerminalTotalDifficulty - } + applyOverrides(newcfg) if err := newcfg.CheckConfigForkOrder(); err != nil { return newcfg, common.Hash{}, err } @@ -301,12 +365,7 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, override // apply the overrides. if genesis == nil && stored != params.MainnetGenesisHash { newcfg = storedcfg - if overrideArrowGlacier != nil { - newcfg.ArrowGlacierBlock = overrideArrowGlacier - } - if overrideTerminalTotalDifficulty != nil { - newcfg.TerminalTotalDifficulty = overrideTerminalTotalDifficulty - } + applyOverrides(newcfg) } // Check config compatibility and write the config. Compatibility errors // are returned to the caller unless we're already at block zero. @@ -322,6 +381,42 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, override return newcfg, stored, nil } +// LoadCliqueConfig loads the stored clique config if the chain config +// is already present in database, otherwise, return the config in the +// provided genesis specification. Note the returned clique config can +// be nil if we are not in the clique network. +func LoadCliqueConfig(db ethdb.Database, genesis *Genesis) (*params.CliqueConfig, error) { + // Load the stored chain config from the database. It can be nil + // in case the database is empty. Notably, we only care about the + // chain config corresponds to the canonical chain. + stored := rawdb.ReadCanonicalHash(db, 0) + if stored != (common.Hash{}) { + storedcfg := rawdb.ReadChainConfig(db, stored) + if storedcfg != nil { + return storedcfg.Clique, nil + } + } + // Load the clique config from the provided genesis specification. + if genesis != nil { + // Reject invalid genesis spec without valid chain config + if genesis.Config == nil { + return nil, errGenesisNoConfig + } + // If the canonical genesis header is present, but the chain + // config is missing(initialize the empty leveldb with an + // external ancient chain segment), ensure the provided genesis + // is matched. + if stored != (common.Hash{}) && genesis.ToBlock().Hash() != stored { + return nil, &GenesisMismatchError{stored, genesis.ToBlock().Hash()} + } + return genesis.Config.Clique, nil + } + // There is no stored chain config and no new config provided, + // In this case the default chain config(mainnet) will be used, + // namely ethash is the specified consensus engine, return nil. + return nil, nil +} + func (g *Genesis) configOrDefault(ghash common.Hash) *params.ChainConfig { switch { case g != nil: @@ -343,13 +438,9 @@ func (g *Genesis) configOrDefault(ghash common.Hash) *params.ChainConfig { } } -// ToBlock creates the genesis block and writes state of a genesis specification -// to the given database (or discards it if nil). -func (g *Genesis) ToBlock(db ethdb.Database) *types.Block { - if db == nil { - db = rawdb.NewMemoryDatabase() - } - root, err := g.Alloc.flush(db) +// ToBlock returns the genesis block according to genesis specification. +func (g *Genesis) ToBlock() *types.Block { + root, err := g.Alloc.deriveHash() if err != nil { panic(err) } @@ -386,7 +477,7 @@ func (g *Genesis) ToBlock(db ethdb.Database) *types.Block { // Commit writes the block and state of a genesis specification to the database. // The block is committed as the canonical head block. func (g *Genesis) Commit(db ethdb.Database) (*types.Block, error) { - block := g.ToBlock(db) + block := g.ToBlock() if block.Number().Sign() != 0 { return nil, errors.New("can't commit genesis block with number > 0") } @@ -400,7 +491,10 @@ func (g *Genesis) Commit(db ethdb.Database) (*types.Block, error) { if config.Clique != nil && len(block.Extra()) < 32+crypto.SignatureLength { return nil, errors.New("can't start clique chain without signers") } - if err := g.Alloc.write(db, block.Hash()); err != nil { + // All the checks has passed, flush the states derived from the genesis + // specification as well as the specification itself into the provided + // database. + if err := g.Alloc.flush(db); err != nil { return nil, err } rawdb.WriteTd(db, block.Hash(), block.NumberU64(), block.Difficulty()) @@ -424,15 +518,6 @@ func (g *Genesis) MustCommit(db ethdb.Database) *types.Block { return block } -// GenesisBlockForTesting creates and writes a block in which addr has the given wei balance. -func GenesisBlockForTesting(db ethdb.Database, addr common.Address, balance *big.Int) *types.Block { - g := Genesis{ - Alloc: GenesisAlloc{addr: {Balance: balance}}, - BaseFee: big.NewInt(params.InitialBaseFee), - } - return g.MustCommit(db) -} - // DefaultGenesisBlock returns the Ethereum main net genesis block. func DefaultGenesisBlock() *Genesis { return &Genesis{ @@ -494,6 +579,7 @@ func DefaultSepoliaGenesisBlock() *Genesis { } } +// DefaultKilnGenesisBlock returns the kiln network genesis block. func DefaultKilnGenesisBlock() *Genesis { g := new(Genesis) reader := strings.NewReader(KilnAllocData) diff --git a/core/genesis_alloc.go b/core/genesis_alloc.go index 041c55424238d..16df390575c21 100644 --- a/core/genesis_alloc.go +++ b/core/genesis_alloc.go @@ -25,7 +25,6 @@ const mainnetAllocData = "\xfa\x04]X\u0793\r\x83b\x011\x8e\u0189\x9agT\x06\x908' const ropstenAllocData = "\xf9\x03\xa4\u0080\x01\xc2\x01\x01\xc2\x02\x01\xc2\x03\x01\xc2\x04\x01\xc2\x05\x01\xc2\x06\x01\xc2\a\x01\xc2\b\x01\xc2\t\x01\xc2\n\x80\xc2\v\x80\xc2\f\x80\xc2\r\x80\xc2\x0e\x80\xc2\x0f\x80\xc2\x10\x80\xc2\x11\x80\xc2\x12\x80\xc2\x13\x80\xc2\x14\x80\xc2\x15\x80\xc2\x16\x80\xc2\x17\x80\xc2\x18\x80\xc2\x19\x80\xc2\x1a\x80\xc2\x1b\x80\xc2\x1c\x80\xc2\x1d\x80\xc2\x1e\x80\xc2\x1f\x80\xc2 \x80\xc2!\x80\xc2\"\x80\xc2#\x80\xc2$\x80\xc2%\x80\xc2&\x80\xc2'\x80\xc2(\x80\xc2)\x80\xc2*\x80\xc2+\x80\xc2,\x80\xc2-\x80\xc2.\x80\xc2/\x80\xc20\x80\xc21\x80\xc22\x80\xc23\x80\xc24\x80\xc25\x80\xc26\x80\xc27\x80\xc28\x80\xc29\x80\xc2:\x80\xc2;\x80\xc2<\x80\xc2=\x80\xc2>\x80\xc2?\x80\xc2@\x80\xc2A\x80\xc2B\x80\xc2C\x80\xc2D\x80\xc2E\x80\xc2F\x80\xc2G\x80\xc2H\x80\xc2I\x80\xc2J\x80\xc2K\x80\xc2L\x80\xc2M\x80\xc2N\x80\xc2O\x80\xc2P\x80\xc2Q\x80\xc2R\x80\xc2S\x80\xc2T\x80\xc2U\x80\xc2V\x80\xc2W\x80\xc2X\x80\xc2Y\x80\xc2Z\x80\xc2[\x80\xc2\\\x80\xc2]\x80\xc2^\x80\xc2_\x80\xc2`\x80\xc2a\x80\xc2b\x80\xc2c\x80\xc2d\x80\xc2e\x80\xc2f\x80\xc2g\x80\xc2h\x80\xc2i\x80\xc2j\x80\xc2k\x80\xc2l\x80\xc2m\x80\xc2n\x80\xc2o\x80\xc2p\x80\xc2q\x80\xc2r\x80\xc2s\x80\xc2t\x80\xc2u\x80\xc2v\x80\xc2w\x80\xc2x\x80\xc2y\x80\xc2z\x80\xc2{\x80\xc2|\x80\xc2}\x80\xc2~\x80\xc2\u007f\x80\u00c1\x80\x80\u00c1\x81\x80\u00c1\x82\x80\u00c1\x83\x80\u00c1\x84\x80\u00c1\x85\x80\u00c1\x86\x80\u00c1\x87\x80\u00c1\x88\x80\u00c1\x89\x80\u00c1\x8a\x80\u00c1\x8b\x80\u00c1\x8c\x80\u00c1\x8d\x80\u00c1\x8e\x80\u00c1\x8f\x80\u00c1\x90\x80\u00c1\x91\x80\u00c1\x92\x80\u00c1\x93\x80\u00c1\x94\x80\u00c1\x95\x80\u00c1\x96\x80\u00c1\x97\x80\u00c1\x98\x80\u00c1\x99\x80\u00c1\x9a\x80\u00c1\x9b\x80\u00c1\x9c\x80\u00c1\x9d\x80\u00c1\x9e\x80\u00c1\x9f\x80\u00c1\xa0\x80\u00c1\xa1\x80\u00c1\xa2\x80\u00c1\xa3\x80\u00c1\xa4\x80\u00c1\xa5\x80\u00c1\xa6\x80\u00c1\xa7\x80\u00c1\xa8\x80\u00c1\xa9\x80\u00c1\xaa\x80\u00c1\xab\x80\u00c1\xac\x80\u00c1\xad\x80\u00c1\xae\x80\u00c1\xaf\x80\u00c1\xb0\x80\u00c1\xb1\x80\u00c1\xb2\x80\u00c1\xb3\x80\u00c1\xb4\x80\u00c1\xb5\x80\u00c1\xb6\x80\u00c1\xb7\x80\u00c1\xb8\x80\u00c1\xb9\x80\u00c1\xba\x80\u00c1\xbb\x80\u00c1\xbc\x80\u00c1\xbd\x80\u00c1\xbe\x80\u00c1\xbf\x80\u00c1\xc0\x80\u00c1\xc1\x80\u00c1\u0080\u00c1\u00c0\u00c1\u0100\u00c1\u0140\u00c1\u0180\u00c1\u01c0\u00c1\u0200\u00c1\u0240\u00c1\u0280\u00c1\u02c0\u00c1\u0300\u00c1\u0340\u00c1\u0380\u00c1\u03c0\u00c1\u0400\u00c1\u0440\u00c1\u0480\u00c1\u04c0\u00c1\u0500\u00c1\u0540\u00c1\u0580\u00c1\u05c0\u00c1\u0600\u00c1\u0640\u00c1\u0680\u00c1\u06c0\u00c1\u0700\u00c1\u0740\u00c1\u0780\u00c1\u07c0\u00c1\xe0\x80\u00c1\xe1\x80\u00c1\xe2\x80\u00c1\xe3\x80\u00c1\xe4\x80\u00c1\xe5\x80\u00c1\xe6\x80\u00c1\xe7\x80\u00c1\xe8\x80\u00c1\xe9\x80\u00c1\xea\x80\u00c1\xeb\x80\u00c1\xec\x80\u00c1\xed\x80\u00c1\xee\x80\u00c1\xef\x80\u00c1\xf0\x80\u00c1\xf1\x80\u00c1\xf2\x80\u00c1\xf3\x80\u00c1\xf4\x80\u00c1\xf5\x80\u00c1\xf6\x80\u00c1\xf7\x80\u00c1\xf8\x80\u00c1\xf9\x80\u00c1\xfa\x80\u00c1\xfb\x80\u00c1\xfc\x80\u00c1\xfd\x80\u00c1\xfe\x80\u00c1\xff\x80\u3507KT\xa8\xbd\x15)f\xd6?pk\xae\x1f\xfe\xb0A\x19!\xe5\x8d\f\x9f,\x9c\xd0Ft\xed\xea@\x00\x00\x00" const rinkebyAllocData = "\xf9\x03\xb7\u0080\x01\xc2\x01\x01\xc2\x02\x01\xc2\x03\x01\xc2\x04\x01\xc2\x05\x01\xc2\x06\x01\xc2\a\x01\xc2\b\x01\xc2\t\x01\xc2\n\x01\xc2\v\x01\xc2\f\x01\xc2\r\x01\xc2\x0e\x01\xc2\x0f\x01\xc2\x10\x01\xc2\x11\x01\xc2\x12\x01\xc2\x13\x01\xc2\x14\x01\xc2\x15\x01\xc2\x16\x01\xc2\x17\x01\xc2\x18\x01\xc2\x19\x01\xc2\x1a\x01\xc2\x1b\x01\xc2\x1c\x01\xc2\x1d\x01\xc2\x1e\x01\xc2\x1f\x01\xc2 \x01\xc2!\x01\xc2\"\x01\xc2#\x01\xc2$\x01\xc2%\x01\xc2&\x01\xc2'\x01\xc2(\x01\xc2)\x01\xc2*\x01\xc2+\x01\xc2,\x01\xc2-\x01\xc2.\x01\xc2/\x01\xc20\x01\xc21\x01\xc22\x01\xc23\x01\xc24\x01\xc25\x01\xc26\x01\xc27\x01\xc28\x01\xc29\x01\xc2:\x01\xc2;\x01\xc2<\x01\xc2=\x01\xc2>\x01\xc2?\x01\xc2@\x01\xc2A\x01\xc2B\x01\xc2C\x01\xc2D\x01\xc2E\x01\xc2F\x01\xc2G\x01\xc2H\x01\xc2I\x01\xc2J\x01\xc2K\x01\xc2L\x01\xc2M\x01\xc2N\x01\xc2O\x01\xc2P\x01\xc2Q\x01\xc2R\x01\xc2S\x01\xc2T\x01\xc2U\x01\xc2V\x01\xc2W\x01\xc2X\x01\xc2Y\x01\xc2Z\x01\xc2[\x01\xc2\\\x01\xc2]\x01\xc2^\x01\xc2_\x01\xc2`\x01\xc2a\x01\xc2b\x01\xc2c\x01\xc2d\x01\xc2e\x01\xc2f\x01\xc2g\x01\xc2h\x01\xc2i\x01\xc2j\x01\xc2k\x01\xc2l\x01\xc2m\x01\xc2n\x01\xc2o\x01\xc2p\x01\xc2q\x01\xc2r\x01\xc2s\x01\xc2t\x01\xc2u\x01\xc2v\x01\xc2w\x01\xc2x\x01\xc2y\x01\xc2z\x01\xc2{\x01\xc2|\x01\xc2}\x01\xc2~\x01\xc2\u007f\x01\u00c1\x80\x01\u00c1\x81\x01\u00c1\x82\x01\u00c1\x83\x01\u00c1\x84\x01\u00c1\x85\x01\u00c1\x86\x01\u00c1\x87\x01\u00c1\x88\x01\u00c1\x89\x01\u00c1\x8a\x01\u00c1\x8b\x01\u00c1\x8c\x01\u00c1\x8d\x01\u00c1\x8e\x01\u00c1\x8f\x01\u00c1\x90\x01\u00c1\x91\x01\u00c1\x92\x01\u00c1\x93\x01\u00c1\x94\x01\u00c1\x95\x01\u00c1\x96\x01\u00c1\x97\x01\u00c1\x98\x01\u00c1\x99\x01\u00c1\x9a\x01\u00c1\x9b\x01\u00c1\x9c\x01\u00c1\x9d\x01\u00c1\x9e\x01\u00c1\x9f\x01\u00c1\xa0\x01\u00c1\xa1\x01\u00c1\xa2\x01\u00c1\xa3\x01\u00c1\xa4\x01\u00c1\xa5\x01\u00c1\xa6\x01\u00c1\xa7\x01\u00c1\xa8\x01\u00c1\xa9\x01\u00c1\xaa\x01\u00c1\xab\x01\u00c1\xac\x01\u00c1\xad\x01\u00c1\xae\x01\u00c1\xaf\x01\u00c1\xb0\x01\u00c1\xb1\x01\u00c1\xb2\x01\u00c1\xb3\x01\u00c1\xb4\x01\u00c1\xb5\x01\u00c1\xb6\x01\u00c1\xb7\x01\u00c1\xb8\x01\u00c1\xb9\x01\u00c1\xba\x01\u00c1\xbb\x01\u00c1\xbc\x01\u00c1\xbd\x01\u00c1\xbe\x01\u00c1\xbf\x01\u00c1\xc0\x01\u00c1\xc1\x01\u00c1\xc2\x01\u00c1\xc3\x01\u00c1\xc4\x01\u00c1\xc5\x01\u00c1\xc6\x01\u00c1\xc7\x01\u00c1\xc8\x01\u00c1\xc9\x01\u00c1\xca\x01\u00c1\xcb\x01\u00c1\xcc\x01\u00c1\xcd\x01\u00c1\xce\x01\u00c1\xcf\x01\u00c1\xd0\x01\u00c1\xd1\x01\u00c1\xd2\x01\u00c1\xd3\x01\u00c1\xd4\x01\u00c1\xd5\x01\u00c1\xd6\x01\u00c1\xd7\x01\u00c1\xd8\x01\u00c1\xd9\x01\u00c1\xda\x01\u00c1\xdb\x01\u00c1\xdc\x01\u00c1\xdd\x01\u00c1\xde\x01\u00c1\xdf\x01\u00c1\xe0\x01\u00c1\xe1\x01\u00c1\xe2\x01\u00c1\xe3\x01\u00c1\xe4\x01\u00c1\xe5\x01\u00c1\xe6\x01\u00c1\xe7\x01\u00c1\xe8\x01\u00c1\xe9\x01\u00c1\xea\x01\u00c1\xeb\x01\u00c1\xec\x01\u00c1\xed\x01\u00c1\xee\x01\u00c1\xef\x01\u00c1\xf0\x01\u00c1\xf1\x01\u00c1\xf2\x01\u00c1\xf3\x01\u00c1\xf4\x01\u00c1\xf5\x01\u00c1\xf6\x01\u00c1\xf7\x01\u00c1\xf8\x01\u00c1\xf9\x01\u00c1\xfa\x01\u00c1\xfb\x01\u00c1\xfc\x01\u00c1\xfd\x01\u00c1\xfe\x01\u00c1\xff\x01\xf6\x941\xb9\x8d\x14\x00{\xde\xe67)\x80\x86\x98\x8a\v\xbd1\x18E#\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" const goerliAllocData = "\xf9\x04\x06\u0080\x01\xc2\x01\x01\xc2\x02\x01\xc2\x03\x01\xc2\x04\x01\xc2\x05\x01\xc2\x06\x01\xc2\a\x01\xc2\b\x01\xc2\t\x01\xc2\n\x01\xc2\v\x01\xc2\f\x01\xc2\r\x01\xc2\x0e\x01\xc2\x0f\x01\xc2\x10\x01\xc2\x11\x01\xc2\x12\x01\xc2\x13\x01\xc2\x14\x01\xc2\x15\x01\xc2\x16\x01\xc2\x17\x01\xc2\x18\x01\xc2\x19\x01\xc2\x1a\x01\xc2\x1b\x01\xc2\x1c\x01\xc2\x1d\x01\xc2\x1e\x01\xc2\x1f\x01\xc2 \x01\xc2!\x01\xc2\"\x01\xc2#\x01\xc2$\x01\xc2%\x01\xc2&\x01\xc2'\x01\xc2(\x01\xc2)\x01\xc2*\x01\xc2+\x01\xc2,\x01\xc2-\x01\xc2.\x01\xc2/\x01\xc20\x01\xc21\x01\xc22\x01\xc23\x01\xc24\x01\xc25\x01\xc26\x01\xc27\x01\xc28\x01\xc29\x01\xc2:\x01\xc2;\x01\xc2<\x01\xc2=\x01\xc2>\x01\xc2?\x01\xc2@\x01\xc2A\x01\xc2B\x01\xc2C\x01\xc2D\x01\xc2E\x01\xc2F\x01\xc2G\x01\xc2H\x01\xc2I\x01\xc2J\x01\xc2K\x01\xc2L\x01\xc2M\x01\xc2N\x01\xc2O\x01\xc2P\x01\xc2Q\x01\xc2R\x01\xc2S\x01\xc2T\x01\xc2U\x01\xc2V\x01\xc2W\x01\xc2X\x01\xc2Y\x01\xc2Z\x01\xc2[\x01\xc2\\\x01\xc2]\x01\xc2^\x01\xc2_\x01\xc2`\x01\xc2a\x01\xc2b\x01\xc2c\x01\xc2d\x01\xc2e\x01\xc2f\x01\xc2g\x01\xc2h\x01\xc2i\x01\xc2j\x01\xc2k\x01\xc2l\x01\xc2m\x01\xc2n\x01\xc2o\x01\xc2p\x01\xc2q\x01\xc2r\x01\xc2s\x01\xc2t\x01\xc2u\x01\xc2v\x01\xc2w\x01\xc2x\x01\xc2y\x01\xc2z\x01\xc2{\x01\xc2|\x01\xc2}\x01\xc2~\x01\xc2\u007f\x01\u00c1\x80\x01\u00c1\x81\x01\u00c1\x82\x01\u00c1\x83\x01\u00c1\x84\x01\u00c1\x85\x01\u00c1\x86\x01\u00c1\x87\x01\u00c1\x88\x01\u00c1\x89\x01\u00c1\x8a\x01\u00c1\x8b\x01\u00c1\x8c\x01\u00c1\x8d\x01\u00c1\x8e\x01\u00c1\x8f\x01\u00c1\x90\x01\u00c1\x91\x01\u00c1\x92\x01\u00c1\x93\x01\u00c1\x94\x01\u00c1\x95\x01\u00c1\x96\x01\u00c1\x97\x01\u00c1\x98\x01\u00c1\x99\x01\u00c1\x9a\x01\u00c1\x9b\x01\u00c1\x9c\x01\u00c1\x9d\x01\u00c1\x9e\x01\u00c1\x9f\x01\u00c1\xa0\x01\u00c1\xa1\x01\u00c1\xa2\x01\u00c1\xa3\x01\u00c1\xa4\x01\u00c1\xa5\x01\u00c1\xa6\x01\u00c1\xa7\x01\u00c1\xa8\x01\u00c1\xa9\x01\u00c1\xaa\x01\u00c1\xab\x01\u00c1\xac\x01\u00c1\xad\x01\u00c1\xae\x01\u00c1\xaf\x01\u00c1\xb0\x01\u00c1\xb1\x01\u00c1\xb2\x01\u00c1\xb3\x01\u00c1\xb4\x01\u00c1\xb5\x01\u00c1\xb6\x01\u00c1\xb7\x01\u00c1\xb8\x01\u00c1\xb9\x01\u00c1\xba\x01\u00c1\xbb\x01\u00c1\xbc\x01\u00c1\xbd\x01\u00c1\xbe\x01\u00c1\xbf\x01\u00c1\xc0\x01\u00c1\xc1\x01\u00c1\xc2\x01\u00c1\xc3\x01\u00c1\xc4\x01\u00c1\xc5\x01\u00c1\xc6\x01\u00c1\xc7\x01\u00c1\xc8\x01\u00c1\xc9\x01\u00c1\xca\x01\u00c1\xcb\x01\u00c1\xcc\x01\u00c1\xcd\x01\u00c1\xce\x01\u00c1\xcf\x01\u00c1\xd0\x01\u00c1\xd1\x01\u00c1\xd2\x01\u00c1\xd3\x01\u00c1\xd4\x01\u00c1\xd5\x01\u00c1\xd6\x01\u00c1\xd7\x01\u00c1\xd8\x01\u00c1\xd9\x01\u00c1\xda\x01\u00c1\xdb\x01\u00c1\xdc\x01\u00c1\xdd\x01\u00c1\xde\x01\u00c1\xdf\x01\u00c1\xe0\x01\u00c1\xe1\x01\u00c1\xe2\x01\u00c1\xe3\x01\u00c1\xe4\x01\u00c1\xe5\x01\u00c1\xe6\x01\u00c1\xe7\x01\u00c1\xe8\x01\u00c1\xe9\x01\u00c1\xea\x01\u00c1\xeb\x01\u00c1\xec\x01\u00c1\xed\x01\u00c1\xee\x01\u00c1\xef\x01\u00c1\xf0\x01\u00c1\xf1\x01\u00c1\xf2\x01\u00c1\xf3\x01\u00c1\xf4\x01\u00c1\xf5\x01\u00c1\xf6\x01\u00c1\xf7\x01\u00c1\xf8\x01\u00c1\xf9\x01\u00c1\xfa\x01\u00c1\xfb\x01\u00c1\xfc\x01\u00c1\xfd\x01\u00c1\xfe\x01\u00c1\xff\x01\xe0\x94L*\xe4\x82Y5\x05\xf0\x16<\xde\xfc\a>\x81\xc6<\xdaA\a\x8a\x15-\x02\xc7\xe1J\xf6\x80\x00\x00\xe0\x94\xa8\xe8\xf1G2e\x8eKQ\xe8q\x191\x05:\x8ai\xba\xf2\xb1\x8a\x15-\x02\xc7\xe1J\xf6\x80\x00\x00\xe1\x94\u0665\x17\x9f\t\x1d\x85\x05\x1d<\x98'\x85\xef\xd1E\\\uc199\x8b\bE\x95\x16\x14\x01HJ\x00\x00\x00\xe1\x94\u08bdBX\xd2v\x887\xba\xa2j(\xfeq\xdc\a\x9f\x84\u01cbJG\xe3\xc1$H\xf4\xad\x00\x00\x00" -const calaverasAllocData = "\xf9\x06\x14\u0080\x01\xc2\x01\x01\xc2\x02\x01\xc2\x03\x01\xc2\x04\x01\xc2\x05\x01\xc2\x06\x01\xc2\a\x01\xc2\b\x01\xc2\t\x01\xc2\n\x01\xc2\v\x01\xc2\f\x01\xc2\r\x01\xc2\x0e\x01\xc2\x0f\x01\xc2\x10\x01\xc2\x11\x01\xc2\x12\x01\xc2\x13\x01\xc2\x14\x01\xc2\x15\x01\xc2\x16\x01\xc2\x17\x01\xc2\x18\x01\xc2\x19\x01\xc2\x1a\x01\xc2\x1b\x01\xc2\x1c\x01\xc2\x1d\x01\xc2\x1e\x01\xc2\x1f\x01\xc2 \x01\xc2!\x01\xc2\"\x01\xc2#\x01\xc2$\x01\xc2%\x01\xc2&\x01\xc2'\x01\xc2(\x01\xc2)\x01\xc2*\x01\xc2+\x01\xc2,\x01\xc2-\x01\xc2.\x01\xc2/\x01\xc20\x01\xc21\x01\xc22\x01\xc23\x01\xc24\x01\xc25\x01\xc26\x01\xc27\x01\xc28\x01\xc29\x01\xc2:\x01\xc2;\x01\xc2<\x01\xc2=\x01\xc2>\x01\xc2?\x01\xc2@\x01\xc2A\x01\xc2B\x01\xc2C\x01\xc2D\x01\xc2E\x01\xc2F\x01\xc2G\x01\xc2H\x01\xc2I\x01\xc2J\x01\xc2K\x01\xc2L\x01\xc2M\x01\xc2N\x01\xc2O\x01\xc2P\x01\xc2Q\x01\xc2R\x01\xc2S\x01\xc2T\x01\xc2U\x01\xc2V\x01\xc2W\x01\xc2X\x01\xc2Y\x01\xc2Z\x01\xc2[\x01\xc2\\\x01\xc2]\x01\xc2^\x01\xc2_\x01\xc2`\x01\xc2a\x01\xc2b\x01\xc2c\x01\xc2d\x01\xc2e\x01\xc2f\x01\xc2g\x01\xc2h\x01\xc2i\x01\xc2j\x01\xc2k\x01\xc2l\x01\xc2m\x01\xc2n\x01\xc2o\x01\xc2p\x01\xc2q\x01\xc2r\x01\xc2s\x01\xc2t\x01\xc2u\x01\xc2v\x01\xc2w\x01\xc2x\x01\xc2y\x01\xc2z\x01\xc2{\x01\xc2|\x01\xc2}\x01\xc2~\x01\xc2\u007f\x01\u00c1\x80\x01\u00c1\x81\x01\u00c1\x82\x01\u00c1\x83\x01\u00c1\x84\x01\u00c1\x85\x01\u00c1\x86\x01\u00c1\x87\x01\u00c1\x88\x01\u00c1\x89\x01\u00c1\x8a\x01\u00c1\x8b\x01\u00c1\x8c\x01\u00c1\x8d\x01\u00c1\x8e\x01\u00c1\x8f\x01\u00c1\x90\x01\u00c1\x91\x01\u00c1\x92\x01\u00c1\x93\x01\u00c1\x94\x01\u00c1\x95\x01\u00c1\x96\x01\u00c1\x97\x01\u00c1\x98\x01\u00c1\x99\x01\u00c1\x9a\x01\u00c1\x9b\x01\u00c1\x9c\x01\u00c1\x9d\x01\u00c1\x9e\x01\u00c1\x9f\x01\u00c1\xa0\x01\u00c1\xa1\x01\u00c1\xa2\x01\u00c1\xa3\x01\u00c1\xa4\x01\u00c1\xa5\x01\u00c1\xa6\x01\u00c1\xa7\x01\u00c1\xa8\x01\u00c1\xa9\x01\u00c1\xaa\x01\u00c1\xab\x01\u00c1\xac\x01\u00c1\xad\x01\u00c1\xae\x01\u00c1\xaf\x01\u00c1\xb0\x01\u00c1\xb1\x01\u00c1\xb2\x01\u00c1\xb3\x01\u00c1\xb4\x01\u00c1\xb5\x01\u00c1\xb6\x01\u00c1\xb7\x01\u00c1\xb8\x01\u00c1\xb9\x01\u00c1\xba\x01\u00c1\xbb\x01\u00c1\xbc\x01\u00c1\xbd\x01\u00c1\xbe\x01\u00c1\xbf\x01\u00c1\xc0\x01\u00c1\xc1\x01\u00c1\xc2\x01\u00c1\xc3\x01\u00c1\xc4\x01\u00c1\xc5\x01\u00c1\xc6\x01\u00c1\xc7\x01\u00c1\xc8\x01\u00c1\xc9\x01\u00c1\xca\x01\u00c1\xcb\x01\u00c1\xcc\x01\u00c1\xcd\x01\u00c1\xce\x01\u00c1\xcf\x01\u00c1\xd0\x01\u00c1\xd1\x01\u00c1\xd2\x01\u00c1\xd3\x01\u00c1\xd4\x01\u00c1\xd5\x01\u00c1\xd6\x01\u00c1\xd7\x01\u00c1\xd8\x01\u00c1\xd9\x01\u00c1\xda\x01\u00c1\xdb\x01\u00c1\xdc\x01\u00c1\xdd\x01\u00c1\xde\x01\u00c1\xdf\x01\u00c1\xe0\x01\u00c1\xe1\x01\u00c1\xe2\x01\u00c1\xe3\x01\u00c1\xe4\x01\u00c1\xe5\x01\u00c1\xe6\x01\u00c1\xe7\x01\u00c1\xe8\x01\u00c1\xe9\x01\u00c1\xea\x01\u00c1\xeb\x01\u00c1\xec\x01\u00c1\xed\x01\u00c1\xee\x01\u00c1\xef\x01\u00c1\xf0\x01\u00c1\xf1\x01\u00c1\xf2\x01\u00c1\xf3\x01\u00c1\xf4\x01\u00c1\xf5\x01\u00c1\xf6\x01\u00c1\xf7\x01\u00c1\xf8\x01\u00c1\xf9\x01\u00c1\xfa\x01\u00c1\xfb\x01\u00c1\xfc\x01\u00c1\xfd\x01\u00c1\xfe\x01\u00c1\xff\x01\xf6\x94\x0e\x89\xe2\xae\xdb\x1c\xfc\u06d4$\xd4\x1a\x1f!\x8fA2s\x81r\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94\x10A\xaf\xbc\xb3Y\u0568\xdcX\xc1[/\xf5\x13T\xff\x8a!}\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94#o\xf1\xe9t\x19\xae\x93\xad\x80\xca\xfb\xaa!\"\f]x\xfb}\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94`\xad\xc0\xf8\x9aA\xaf#|\xe75T\xed\xe1p\xd73\xec\x14\xe0\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94y\x9d2\x9e_X4\x19\x16|\xd7\"\x96$\x85\x92n3\x8fJ\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94|\xf5\xb7\x9b\xfe)\x1ag\xab\x02\xb3\x93\xe4V\xcc\xc4\xc2f\xf7S\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94\x8a\x8e\xaf\xb1\xcfb\xbf\xbe\xb1t\x17i\xda\xe1\xa9\xddG\x99a\x92\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94\x8b\xa1\xf1\tU\x1b\xd42\x800\x12dZ\xc16\xdd\xd6M\xbar\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94\xb0*.\xda\x1b1\u007f\xbd\x16v\x01(\x83k\n\u015bV\x0e\x9d\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94\xba\xdc\r\xe9\xe0yK\x04\x9b^\xa6<>\x1ei\x8a4v\xc1r\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94\xf00\v\ue24a\xe2r\xeb4~\x83i\xac\fv\xdfB\xc9?\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94\xfe;U~\x8f\xb6+\x89\xf4\x91kr\x1b\xe5\\\ub08d\xbds\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" const sepoliaAllocData = "\xf9\x01\xee\u0791i\x16\xa8{\x823?BE\x04f#\xb27\x94\xc6\\\x8b\bE\x95\x16\x14\x01HJ\x00\x00\x00\xe1\x94\x10\xf5\xd4XT\xe08\a\x14\x85\xac\x9e@#\b\u03c0\xd2\xd2\xfe\x8bR\xb7\xd2\xdc\xc8\f\xd2\xe4\x00\x00\x00\u0794y\x9d2\x9e_X4\x19\x16|\xd7\"\x96$\x85\x92n3\x8fJ\x88\r\u0db3\xa7d\x00\x00\xe0\x94|\xf5\xb7\x9b\xfe)\x1ag\xab\x02\xb3\x93\xe4V\xcc\xc4\xc2f\xf7S\x8a\xd3\xc2\x1b\xce\xcc\xed\xa1\x00\x00\x00\xe0\x94\x8b\u007f\tw\xbbO\x0f\xbepv\xfa\"\xbc$\xac\xa0CX?^\x8a\xd3\xc2\x1b\xce\xcc\xed\xa1\x00\x00\x00\xe0\x94\xa2\xa6\xd949\x14O\xfeM'\xc9\xe0\x88\xdc\u0637\x83\x94bc\x8a\xd3\xc2\x1b\xce\xcc\xed\xa1\x00\x00\x00\xe0\x94\xaa\xec\x869DA\xf9\x15\xbc\xe3\xe6\xab9\x99w\xe9\x90o;i\x8a\xd3\xc2\x1b\xce\xcc\xed\xa1\x00\x00\x00\u1532\x1c3\xde\x1f\xab?\xa1T\x99\xc6+Y\xfe\f\xc3%\x00 \u044bR\xb7\xd2\xdc\xc8\f\xd2\xe4\x00\x00\x00\xe0\x94\xbc\x11)Y6\xaay\u0554\x13\x9d\xe1\xb2\xe1&)AO;\u06ca\xd3\xc2\x1b\xce\xcc\xed\xa1\x00\x00\x00\xe0\x94\xbe\xef2\xca[\x9a\x19\x8d'\xb4\xe0/LpC\x9f\xe6\x03V\u03ca\xd3\xc2\x1b\xce\xcc\xed\xa1\x00\x00\x00\xe1\x94\xd7\xd7lX\xb3\xa5\x19\xe9\xfal\xc4\xd2-\xc0\x17%\x9b\u011f\x1e\x8bR\xb7\xd2\xdc\xc8\f\xd2\xe4\x00\x00\x00\xe0\x94\xd7\xed\xdbx\xed)[<\x96)$\x0e\x89$\xfb\x8d\x88t\xdd\u060a\xd3\xc2\x1b\xce\xcc\xed\xa1\x00\x00\x00\xe0\x94\u0665\x17\x9f\t\x1d\x85\x05\x1d<\x98'\x85\xef\xd1E\\\uc199\x8a\xd3\xc2\x1b\xce\xcc\xed\xa1\x00\x00\x00\xe0\x94\xe2\xe2e\x90(\x147\x84\xd5W\xbc\xeco\xf3\xa0r\x10H\x88\n\x8a\xd3\xc2\x1b\xce\xcc\xed\xa1\x00\x00\x00\xe0\x94\xf4|\xae\x1c\xf7\x9c\xa6u\x8b\xfcx}\xbd!\u6f7eq\x12\xb8\x8a\xd3\xc2\x1b\xce\xcc\xed\xa1\x00\x00\x00" const KilnAllocData = `{ "config": { diff --git a/core/genesis_test.go b/core/genesis_test.go index e8010e3d4ebd4..a7d04f53fe23a 100644 --- a/core/genesis_test.go +++ b/core/genesis_test.go @@ -117,7 +117,7 @@ func TestSetupGenesis(t *testing.T) { // Advance to block #4, past the homestead transition block of customg. genesis := oldcustomg.MustCommit(db) - bc, _ := NewBlockChain(db, nil, oldcustomg.Config, ethash.NewFullFaker(), vm.Config{}, nil, nil) + bc, _ := NewBlockChain(db, nil, &oldcustomg, nil, ethash.NewFullFaker(), vm.Config{}, nil, nil) defer bc.Stop() blocks, _ := GenerateChain(oldcustomg.Config, genesis, ethash.NewFaker(), db, 4, nil) @@ -178,7 +178,7 @@ func TestGenesisHashes(t *testing.T) { t.Errorf("case: %d a), want: %s, got: %s", i, c.want.Hex(), have.Hex()) } // Test via ToBlock - if have := c.genesis.ToBlock(nil).Hash(); have != c.want { + if have := c.genesis.ToBlock().Hash(); have != c.want { t.Errorf("case: %d a), want: %s, got: %s", i, c.want.Hex(), have.Hex()) } } @@ -192,11 +192,7 @@ func TestGenesis_Commit(t *testing.T) { } db := rawdb.NewMemoryDatabase() - genesisBlock, err := genesis.Commit(db) - if err != nil { - t.Fatal(err) - } - + genesisBlock := genesis.MustCommit(db) if genesis.Difficulty != nil { t.Fatalf("assumption wrong") } @@ -221,12 +217,12 @@ func TestReadWriteGenesisAlloc(t *testing.T) { {1}: {Balance: big.NewInt(1), Storage: map[common.Hash]common.Hash{{1}: {1}}}, {2}: {Balance: big.NewInt(2), Storage: map[common.Hash]common.Hash{{2}: {2}}}, } - hash = common.HexToHash("0xdeadbeef") + hash, _ = alloc.deriveHash() ) - alloc.write(db, hash) + alloc.flush(db) var reload GenesisAlloc - err := reload.UnmarshalJSON(rawdb.ReadGenesisState(db, hash)) + err := reload.UnmarshalJSON(rawdb.ReadGenesisStateSpec(db, hash)) if err != nil { t.Fatalf("Failed to load genesis state %v", err) } diff --git a/core/headerchain_test.go b/core/headerchain_test.go index ed0522671fb81..fe083b0031451 100644 --- a/core/headerchain_test.go +++ b/core/headerchain_test.go @@ -27,7 +27,6 @@ import ( "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" ) @@ -70,19 +69,18 @@ func testInsert(t *testing.T, hc *HeaderChain, chain []*types.Header, wantStatus // This test checks status reporting of InsertHeaderChain. func TestHeaderInsertion(t *testing.T) { var ( - db = rawdb.NewMemoryDatabase() - genesis = (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(db) + db = rawdb.NewMemoryDatabase() + gspec = &Genesis{BaseFee: big.NewInt(params.InitialBaseFee), Config: params.AllEthashProtocolChanges} ) - - hc, err := NewHeaderChain(db, params.AllEthashProtocolChanges, ethash.NewFaker(), func() bool { return false }) + gspec.Commit(db) + hc, err := NewHeaderChain(db, gspec.Config, ethash.NewFaker(), func() bool { return false }) if err != nil { t.Fatal(err) } // chain A: G->A1->A2...A128 - chainA := makeHeaderChain(genesis.Header(), 128, ethash.NewFaker(), db, 10) + genDb, chainA := makeHeaderChainWithGenesis(gspec, 128, ethash.NewFaker(), 10) // chain B: G->A1->B1...B128 - chainB := makeHeaderChain(chainA[0], 128, ethash.NewFaker(), db, 10) - log.Root().SetHandler(log.StdoutHandler) + chainB := makeHeaderChain(gspec.Config, chainA[0], 128, ethash.NewFaker(), genDb, 10) forker := NewForkChoice(hc, nil) // Inserting 64 headers on an empty chain, expecting diff --git a/core/mkalloc.go b/core/mkalloc.go index df167d7082cd0..e4c2ec0d83e9f 100644 --- a/core/mkalloc.go +++ b/core/mkalloc.go @@ -18,12 +18,10 @@ // +build none /* +The mkalloc tool creates the genesis allocation constants in genesis_alloc.go +It outputs a const declaration that contains an RLP-encoded list of (address, balance) tuples. - The mkalloc tool creates the genesis allocation constants in genesis_alloc.go - It outputs a const declaration that contains an RLP-encoded list of (address, balance) tuples. - - go run mkalloc.go genesis.json - + go run mkalloc.go genesis.json */ package main diff --git a/core/rawdb/accessors_chain.go b/core/rawdb/accessors_chain.go index 8ea2e2ca72730..881660aa8e8fd 100644 --- a/core/rawdb/accessors_chain.go +++ b/core/rawdb/accessors_chain.go @@ -37,7 +37,7 @@ import ( func ReadCanonicalHash(db ethdb.Reader, number uint64) common.Hash { var data []byte db.ReadAncients(func(reader ethdb.AncientReaderOp) error { - data, _ = reader.Ancient(freezerHashTable, number) + data, _ = reader.Ancient(chainFreezerHashTable, number) if len(data) == 0 { // Get it by hash from leveldb data, _ = db.Get(headerHashKey(number)) @@ -259,8 +259,7 @@ func WriteLastPivotNumber(db ethdb.KeyValueWriter, pivot uint64) { } // ReadTxIndexTail retrieves the number of oldest indexed block -// whose transaction indices has been indexed. If the corresponding entry -// is non-existent in database it means the indexing has been finished. +// whose transaction indices has been indexed. func ReadTxIndexTail(db ethdb.KeyValueReader) *uint64 { data, _ := db.Get(txIndexTailKey) if len(data) != 8 { @@ -335,7 +334,7 @@ func ReadHeaderRange(db ethdb.Reader, number uint64, count uint64) []rlp.RawValu } // read remaining from ancients max := count * 700 - data, err := db.AncientRange(freezerHeaderTable, i+1-count, count, max) + data, err := db.AncientRange(chainFreezerHeaderTable, i+1-count, count, max) if err == nil && uint64(len(data)) == count { // the data is on the order [h, h+1, .., n] -- reordering needed for i := range data { @@ -352,7 +351,7 @@ func ReadHeaderRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValu // First try to look up the data in ancient database. Extra hash // comparison is necessary since ancient database only maintains // the canonical data. - data, _ = reader.Ancient(freezerHeaderTable, number) + data, _ = reader.Ancient(chainFreezerHeaderTable, number) if len(data) > 0 && crypto.Keccak256Hash(data) == hash { return nil } @@ -428,7 +427,7 @@ func deleteHeaderWithoutNumber(db ethdb.KeyValueWriter, hash common.Hash, number // isCanon is an internal utility method, to check whether the given number/hash // is part of the ancient (canon) set. func isCanon(reader ethdb.AncientReaderOp, number uint64, hash common.Hash) bool { - h, err := reader.Ancient(freezerHashTable, number) + h, err := reader.Ancient(chainFreezerHashTable, number) if err != nil { return false } @@ -444,7 +443,7 @@ func ReadBodyRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValue db.ReadAncients(func(reader ethdb.AncientReaderOp) error { // Check if the data is in ancients if isCanon(reader, number, hash) { - data, _ = reader.Ancient(freezerBodiesTable, number) + data, _ = reader.Ancient(chainFreezerBodiesTable, number) return nil } // If not, try reading from leveldb @@ -459,7 +458,7 @@ func ReadBodyRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValue func ReadCanonicalBodyRLP(db ethdb.Reader, number uint64) rlp.RawValue { var data []byte db.ReadAncients(func(reader ethdb.AncientReaderOp) error { - data, _ = reader.Ancient(freezerBodiesTable, number) + data, _ = reader.Ancient(chainFreezerBodiesTable, number) if len(data) > 0 { return nil } @@ -527,7 +526,7 @@ func ReadTdRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValue { db.ReadAncients(func(reader ethdb.AncientReaderOp) error { // Check if the data is in ancients if isCanon(reader, number, hash) { - data, _ = reader.Ancient(freezerDifficultyTable, number) + data, _ = reader.Ancient(chainFreezerDifficultyTable, number) return nil } // If not, try reading from leveldb @@ -587,7 +586,7 @@ func ReadReceiptsRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawVa db.ReadAncients(func(reader ethdb.AncientReaderOp) error { // Check if the data is in ancients if isCanon(reader, number, hash) { - data, _ = reader.Ancient(freezerReceiptTable, number) + data, _ = reader.Ancient(chainFreezerReceiptTable, number) return nil } // If not, try reading from leveldb @@ -819,19 +818,19 @@ func WriteAncientBlocks(db ethdb.AncientWriter, blocks []*types.Block, receipts func writeAncientBlock(op ethdb.AncientWriteOp, block *types.Block, header *types.Header, receipts []*types.ReceiptForStorage, td *big.Int) error { num := block.NumberU64() - if err := op.AppendRaw(freezerHashTable, num, block.Hash().Bytes()); err != nil { + if err := op.AppendRaw(chainFreezerHashTable, num, block.Hash().Bytes()); err != nil { return fmt.Errorf("can't add block %d hash: %v", num, err) } - if err := op.Append(freezerHeaderTable, num, header); err != nil { + if err := op.Append(chainFreezerHeaderTable, num, header); err != nil { return fmt.Errorf("can't append block header %d: %v", num, err) } - if err := op.Append(freezerBodiesTable, num, block.Body()); err != nil { + if err := op.Append(chainFreezerBodiesTable, num, block.Body()); err != nil { return fmt.Errorf("can't append block body %d: %v", num, err) } - if err := op.Append(freezerReceiptTable, num, receipts); err != nil { + if err := op.Append(chainFreezerReceiptTable, num, receipts); err != nil { return fmt.Errorf("can't append block %d receipts: %v", num, err) } - if err := op.Append(freezerDifficultyTable, num, td); err != nil { + if err := op.Append(chainFreezerDifficultyTable, num, td); err != nil { return fmt.Errorf("can't append block %d total difficulty: %v", num, err) } return nil diff --git a/core/rawdb/accessors_chain_test.go b/core/rawdb/accessors_chain_test.go index dbb13caa416c5..21d23e1f0c8b8 100644 --- a/core/rawdb/accessors_chain_test.go +++ b/core/rawdb/accessors_chain_test.go @@ -285,7 +285,7 @@ func TestTdStorage(t *testing.T) { func TestCanonicalMappingStorage(t *testing.T) { db := NewMemoryDatabase() - // Create a test canonical number and assinged hash to move around + // Create a test canonical number and assigned hash to move around hash, number := common.Hash{0: 0xff}, uint64(314) if entry := ReadCanonicalHash(db, number); entry != (common.Hash{}) { t.Fatalf("Non existent canonical mapping returned: %v", entry) diff --git a/core/rawdb/accessors_metadata.go b/core/rawdb/accessors_metadata.go index f5a161adb6889..7a9e6442f0115 100644 --- a/core/rawdb/accessors_metadata.go +++ b/core/rawdb/accessors_metadata.go @@ -81,15 +81,16 @@ func WriteChainConfig(db ethdb.KeyValueWriter, hash common.Hash, cfg *params.Cha } } -// ReadGenesisState retrieves the genesis state based on the given genesis hash. -func ReadGenesisState(db ethdb.KeyValueReader, hash common.Hash) []byte { - data, _ := db.Get(genesisKey(hash)) +// ReadGenesisStateSpec retrieves the genesis state specification based on the +// given genesis hash. +func ReadGenesisStateSpec(db ethdb.KeyValueReader, hash common.Hash) []byte { + data, _ := db.Get(genesisStateSpecKey(hash)) return data } -// WriteGenesisState writes the genesis state into the disk. -func WriteGenesisState(db ethdb.KeyValueWriter, hash common.Hash, data []byte) { - if err := db.Put(genesisKey(hash), data); err != nil { +// WriteGenesisStateSpec writes the genesis state specification into the disk. +func WriteGenesisStateSpec(db ethdb.KeyValueWriter, hash common.Hash, data []byte) { + if err := db.Put(genesisStateSpecKey(hash), data); err != nil { log.Crit("Failed to store genesis state", "err", err) } } diff --git a/core/rawdb/ancient_scheme.go b/core/rawdb/ancient_scheme.go new file mode 100644 index 0000000000000..3da061cbd977c --- /dev/null +++ b/core/rawdb/ancient_scheme.go @@ -0,0 +1,86 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rawdb + +import "fmt" + +// The list of table names of chain freezer. +const ( + // chainFreezerHeaderTable indicates the name of the freezer header table. + chainFreezerHeaderTable = "headers" + + // chainFreezerHashTable indicates the name of the freezer canonical hash table. + chainFreezerHashTable = "hashes" + + // chainFreezerBodiesTable indicates the name of the freezer block body table. + chainFreezerBodiesTable = "bodies" + + // chainFreezerReceiptTable indicates the name of the freezer receipts table. + chainFreezerReceiptTable = "receipts" + + // chainFreezerDifficultyTable indicates the name of the freezer total difficulty table. + chainFreezerDifficultyTable = "diffs" +) + +// chainFreezerNoSnappy configures whether compression is disabled for the ancient-tables. +// Hashes and difficulties don't compress well. +var chainFreezerNoSnappy = map[string]bool{ + chainFreezerHeaderTable: false, + chainFreezerHashTable: true, + chainFreezerBodiesTable: false, + chainFreezerReceiptTable: false, + chainFreezerDifficultyTable: true, +} + +// The list of identifiers of ancient stores. +var ( + chainFreezerName = "chain" // the folder name of chain segment ancient store. +) + +// freezers the collections of all builtin freezers. +var freezers = []string{chainFreezerName} + +// InspectFreezerTable dumps out the index of a specific freezer table. The passed +// ancient indicates the path of root ancient directory where the chain freezer can +// be opened. Start and end specify the range for dumping out indexes. +// Note this function can only be used for debugging purposes. +func InspectFreezerTable(ancient string, freezerName string, tableName string, start, end int64) error { + var ( + path string + tables map[string]bool + ) + switch freezerName { + case chainFreezerName: + path, tables = resolveChainFreezerDir(ancient), chainFreezerNoSnappy + default: + return fmt.Errorf("unknown freezer, supported ones: %v", freezers) + } + noSnappy, exist := tables[tableName] + if !exist { + var names []string + for name := range tables { + names = append(names, name) + } + return fmt.Errorf("unknown table, supported ones: %v", names) + } + table, err := newFreezerTable(path, tableName, noSnappy, true) + if err != nil { + return err + } + table.dumpIndexStdout(start, end) + return nil +} diff --git a/core/rawdb/chain_freezer.go b/core/rawdb/chain_freezer.go index 4c49db2748b22..212ec73ed73d2 100644 --- a/core/rawdb/chain_freezer.go +++ b/core/rawdb/chain_freezer.go @@ -92,6 +92,8 @@ func (f *chainFreezer) freeze(db ethdb.KeyValueStore) { backoff bool triggered chan struct{} // Used in tests ) + timer := time.NewTimer(freezerRecheckInterval) + defer timer.Stop() for { select { case <-f.quit: @@ -106,8 +108,9 @@ func (f *chainFreezer) freeze(db ethdb.KeyValueStore) { triggered = nil } select { - case <-time.NewTimer(freezerRecheckInterval).C: + case <-timer.C: backoff = false + timer.Reset(freezerRecheckInterval) case triggered = <-f.trigger: backoff = false case <-f.quit: @@ -241,7 +244,7 @@ func (f *chainFreezer) freeze(db ethdb.KeyValueStore) { if n := len(ancients); n > 0 { context = append(context, []interface{}{"hash", ancients[n-1]}...) } - log.Info("Deep froze chain segment", context...) + log.Debug("Deep froze chain segment", context...) // Avoid database thrashing with tiny writes if frozen-first < freezerBatchLimit { @@ -278,19 +281,19 @@ func (f *chainFreezer) freezeRange(nfdb *nofreezedb, number, limit uint64) (hash } // Write to the batch. - if err := op.AppendRaw(freezerHashTable, number, hash[:]); err != nil { + if err := op.AppendRaw(chainFreezerHashTable, number, hash[:]); err != nil { return fmt.Errorf("can't write hash to Freezer: %v", err) } - if err := op.AppendRaw(freezerHeaderTable, number, header); err != nil { + if err := op.AppendRaw(chainFreezerHeaderTable, number, header); err != nil { return fmt.Errorf("can't write header to Freezer: %v", err) } - if err := op.AppendRaw(freezerBodiesTable, number, body); err != nil { + if err := op.AppendRaw(chainFreezerBodiesTable, number, body); err != nil { return fmt.Errorf("can't write body to Freezer: %v", err) } - if err := op.AppendRaw(freezerReceiptTable, number, receipts); err != nil { + if err := op.AppendRaw(chainFreezerReceiptTable, number, receipts); err != nil { return fmt.Errorf("can't write receipts to Freezer: %v", err) } - if err := op.AppendRaw(freezerDifficultyTable, number, td); err != nil { + if err := op.AppendRaw(chainFreezerDifficultyTable, number, td); err != nil { return fmt.Errorf("can't write td to Freezer: %v", err) } diff --git a/core/rawdb/chain_iterator.go b/core/rawdb/chain_iterator.go index 21e42f42d43ac..121f6d39dda33 100644 --- a/core/rawdb/chain_iterator.go +++ b/core/rawdb/chain_iterator.go @@ -50,7 +50,7 @@ func InitDatabaseFromFreezer(db ethdb.Database) { if i+count > frozen { count = frozen - i } - data, err := db.AncientRange(freezerHashTable, i, count, 32*count) + data, err := db.AncientRange(chainFreezerHashTable, i, count, 32*count) if err != nil { log.Crit("Failed to init database from freezer", "err", err) } @@ -243,7 +243,7 @@ func indexTransactions(db ethdb.Database, from uint64, to uint64, interrupt chan case <-interrupt: log.Debug("Transaction indexing interrupted", "blocks", blocks, "txs", txs, "tail", lastNum, "elapsed", common.PrettyDuration(time.Since(start))) default: - log.Info("Indexed transactions", "blocks", blocks, "txs", txs, "tail", lastNum, "elapsed", common.PrettyDuration(time.Since(start))) + log.Debug("Indexed transactions", "blocks", blocks, "txs", txs, "tail", lastNum, "elapsed", common.PrettyDuration(time.Since(start))) } } @@ -335,7 +335,7 @@ func unindexTransactions(db ethdb.Database, from uint64, to uint64, interrupt ch case <-interrupt: log.Debug("Transaction unindexing interrupted", "blocks", blocks, "txs", txs, "tail", to, "elapsed", common.PrettyDuration(time.Since(start))) default: - log.Info("Unindexed transactions", "blocks", blocks, "txs", txs, "tail", to, "elapsed", common.PrettyDuration(time.Since(start))) + log.Debug("Unindexed transactions", "blocks", blocks, "txs", txs, "tail", to, "elapsed", common.PrettyDuration(time.Since(start))) } } diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 2b870d16d44f9..1eaf033bbefa5 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -21,6 +21,7 @@ import ( "errors" "fmt" "os" + "path" "sync/atomic" "time" @@ -34,10 +35,16 @@ import ( // freezerdb is a database wrapper that enabled freezer data retrievals. type freezerdb struct { + ancientRoot string ethdb.KeyValueStore ethdb.AncientStore } +// AncientDatadir returns the path of root ancient directory. +func (frdb *freezerdb) AncientDatadir() (string, error) { + return frdb.ancientRoot, nil +} + // Close implements io.Closer, closing both the fast key-value store as well as // the slow ancient tables. func (frdb *freezerdb) Close() error { @@ -162,12 +169,36 @@ func NewDatabase(db ethdb.KeyValueStore) ethdb.Database { return &nofreezedb{KeyValueStore: db} } +// resolveChainFreezerDir is a helper function which resolves the absolute path +// of chain freezer by considering backward compatibility. +func resolveChainFreezerDir(ancient string) string { + // Check if the chain freezer is already present in the specified + // sub folder, if not then two possibilities: + // - chain freezer is not initialized + // - chain freezer exists in legacy location (root ancient folder) + freezer := path.Join(ancient, chainFreezerName) + if !common.FileExist(freezer) { + if !common.FileExist(ancient) { + // The entire ancient store is not initialized, still use the sub + // folder for initialization. + } else { + // Ancient root is already initialized, then we hold the assumption + // that chain freezer is also initialized and located in root folder. + // In this case fallback to legacy location. + freezer = ancient + log.Info("Found legacy ancient chain path", "location", ancient) + } + } + return freezer +} + // NewDatabaseWithFreezer creates a high level database on top of a given key- // value data store with a freezer moving immutable chain segments into cold -// storage. -func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace string, readonly bool) (ethdb.Database, error) { +// storage. The passed ancient indicates the path of root ancient directory +// where the chain freezer can be opened. +func NewDatabaseWithFreezer(db ethdb.KeyValueStore, ancient string, namespace string, readonly bool) (ethdb.Database, error) { // Create the idle freezer instance - frdb, err := newChainFreezer(freezer, namespace, readonly, freezerTableSize, FreezerNoSnappy) + frdb, err := newChainFreezer(resolveChainFreezerDir(ancient), namespace, readonly, freezerTableSize, chainFreezerNoSnappy) if err != nil { return nil, err } @@ -198,7 +229,7 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace st // If the freezer already contains something, ensure that the genesis blocks // match, otherwise we might mix up freezers across chains and destroy both // the freezer and the key-value store. - frgenesis, err := frdb.Ancient(freezerHashTable, 0) + frgenesis, err := frdb.Ancient(chainFreezerHashTable, 0) if err != nil { return nil, fmt.Errorf("failed to retrieve genesis from ancient %v", err) } else if !bytes.Equal(kvgenesis, frgenesis) { @@ -208,7 +239,7 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace st // are contiguous, otherwise we might end up with a non-functional freezer. if kvhash, _ := db.Get(headerHashKey(frozen)); len(kvhash) == 0 { // Subsequent header after the freezer limit is missing from the database. - // Reject startup is the database has a more recent head. + // Reject startup if the database has a more recent head. if *ReadHeaderNumber(db, ReadHeadHeaderHash(db)) > frozen-1 { return nil, fmt.Errorf("gap (#%d) in the chain between ancients and leveldb", frozen) } @@ -229,7 +260,7 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace st if kvblob, _ := db.Get(headerHashKey(1)); len(kvblob) == 0 { return nil, errors.New("ancient chain segments already extracted, please set --datadir.ancient to the correct path") } - // Block #1 is still in the database, we're allowed to init a new feezer + // Block #1 is still in the database, we're allowed to init a new freezer } // Otherwise, the head header is still the genesis, we're allowed to init a new // freezer. @@ -244,6 +275,7 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace st }() } return &freezerdb{ + ancientRoot: ancient, KeyValueStore: db, AncientStore: frdb, }, nil @@ -273,13 +305,15 @@ func NewLevelDBDatabase(file string, cache int, handles int, namespace string, r } // NewLevelDBDatabaseWithFreezer creates a persistent key-value database with a -// freezer moving immutable chain segments into cold storage. -func NewLevelDBDatabaseWithFreezer(file string, cache int, handles int, freezer string, namespace string, readonly bool) (ethdb.Database, error) { +// freezer moving immutable chain segments into cold storage. The passed ancient +// indicates the path of root ancient directory where the chain freezer can be +// opened. +func NewLevelDBDatabaseWithFreezer(file string, cache int, handles int, ancient string, namespace string, readonly bool) (ethdb.Database, error) { kvdb, err := leveldb.New(file, cache, handles, namespace, readonly) if err != nil { return nil, err } - frdb, err := NewDatabaseWithFreezer(kvdb, freezer, namespace, readonly) + frdb, err := NewDatabaseWithFreezer(kvdb, ancient, namespace, readonly) if err != nil { kvdb.Close() return nil, err @@ -441,7 +475,7 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { } // Inspect append-only file store then. ancientSizes := []*common.StorageSize{&ancientHeadersSize, &ancientBodiesSize, &ancientReceiptsSize, &ancientHashesSize, &ancientTdsSize} - for i, category := range []string{freezerHeaderTable, freezerBodiesTable, freezerReceiptTable, freezerHashTable, freezerDifficultyTable} { + for i, category := range []string{chainFreezerHeaderTable, chainFreezerBodiesTable, chainFreezerReceiptTable, chainFreezerHashTable, chainFreezerDifficultyTable} { if size, err := db.AncientSize(category); err == nil { *ancientSizes[i] += common.StorageSize(size) total += common.StorageSize(size) diff --git a/core/rawdb/freezer.go b/core/rawdb/freezer.go index 63fd8cdcf86f5..53bd989a482d8 100644 --- a/core/rawdb/freezer.go +++ b/core/rawdb/freezer.go @@ -57,10 +57,10 @@ const freezerTableSize = 2 * 1000 * 1000 * 1000 // Freezer is a memory mapped append-only database to store immutable ordered // data into flat files: // -// - The append-only nature ensures that disk writes are minimized. -// - The memory mapping ensures we can max out system memory for caching without -// reserving it for go-ethereum. This would also reduce the memory requirements -// of Geth, and thus also GC overhead. +// - The append-only nature ensures that disk writes are minimized. +// - The memory mapping ensures we can max out system memory for caching without +// reserving it for go-ethereum. This would also reduce the memory requirements +// of Geth, and thus also GC overhead. type Freezer struct { // WARNING: The `frozen` and `tail` fields are accessed atomically. On 32 bit platforms, only // 64-bit aligned fields can be atomic. The struct is guaranteed to be so aligned, @@ -68,8 +68,6 @@ type Freezer struct { frozen uint64 // Number of blocks already frozen tail uint64 // Number of the first stored item in the freezer - datadir string // Path of root directory of ancient store - // This lock synchronizes writers and the truncate operation, as well as // the "atomic" (batched) read operations. writeLock sync.RWMutex @@ -111,7 +109,6 @@ func NewFreezer(datadir string, namespace string, readonly bool, maxTableSize ui readonly: readonly, tables: make(map[string]*freezerTable), instanceLock: lock, - datadir: datadir, } // Create the tables. @@ -191,9 +188,9 @@ func (f *Freezer) Ancient(kind string, number uint64) ([]byte, error) { // AncientRange retrieves multiple items in sequence, starting from the index 'start'. // It will return -// - at most 'max' items, -// - at least 1 item (even if exceeding the maxByteSize), but will otherwise -// return as many items as fit into maxByteSize. +// - at most 'max' items, +// - at least 1 item (even if exceeding the maxByteSize), but will otherwise +// return as many items as fit into maxByteSize. func (f *Freezer) AncientRange(kind string, start, count, maxBytes uint64) ([][]byte, error) { if table := f.tables[kind]; table != nil { return table.RetrieveItems(start, count, maxBytes) @@ -429,7 +426,7 @@ func (f *Freezer) MigrateTable(kind string, convert convertLegacyFn) error { // Set up new dir for the migrated table, the content of which // we'll at the end move over to the ancients dir. migrationPath := filepath.Join(ancientsPath, "migration") - newTable, err := NewFreezerTable(migrationPath, kind, table.noCompression, false) + newTable, err := newFreezerTable(migrationPath, kind, table.noCompression, false) if err != nil { return err } @@ -486,11 +483,5 @@ func (f *Freezer) MigrateTable(kind string, convert convertLegacyFn) error { if err := os.Remove(migrationPath); err != nil { return err } - return nil } - -// AncientDatadir returns the root directory path of the ancient store. -func (f *Freezer) AncientDatadir() (string, error) { - return f.datadir, nil -} diff --git a/core/rawdb/freezer_table.go b/core/rawdb/freezer_table.go index dd4a80efcbc57..3fe691cf6d2a7 100644 --- a/core/rawdb/freezer_table.go +++ b/core/rawdb/freezer_table.go @@ -46,7 +46,7 @@ var ( errNotSupported = errors.New("this operation is not supported") ) -// indexEntry contains the number/id of the file that the data resides in, aswell as the +// indexEntry contains the number/id of the file that the data resides in, as well as the // offset within the file to the end of the data. // In serialized form, the filenum is stored as uint16. type indexEntry struct { @@ -123,8 +123,8 @@ type freezerTable struct { lock sync.RWMutex // Mutex protecting the data file descriptors } -// NewFreezerTable opens the given path as a freezer table. -func NewFreezerTable(path, name string, disableSnappy, readonly bool) (*freezerTable, error) { +// newFreezerTable opens the given path as a freezer table. +func newFreezerTable(path, name string, disableSnappy, readonly bool) (*freezerTable, error) { return newTable(path, name, metrics.NilMeter{}, metrics.NilMeter{}, metrics.NilGauge{}, freezerTableSize, disableSnappy, readonly) } @@ -884,9 +884,7 @@ func (t *freezerTable) Sync() error { return t.head.Sync() } -// DumpIndex is a debug print utility function, mainly for testing. It can also -// be used to analyse a live freezer table index. -func (t *freezerTable) DumpIndex(start, stop int64) { +func (t *freezerTable) dumpIndexStdout(start, stop int64) { t.dumpIndex(os.Stdout, start, stop) } diff --git a/core/rawdb/freezer_table_test.go b/core/rawdb/freezer_table_test.go index 0bddcf7211363..ea28e71756de6 100644 --- a/core/rawdb/freezer_table_test.go +++ b/core/rawdb/freezer_table_test.go @@ -902,7 +902,7 @@ func TestSequentialRead(t *testing.T) { } // Write 15 bytes 30 times writeChunks(t, f, 30, 15) - f.DumpIndex(0, 30) + f.dumpIndexStdout(0, 30) f.Close() } { // Open it, iterate, verify iteration diff --git a/core/rawdb/freezer_test.go b/core/rawdb/freezer_test.go index c28d35ef387d0..630911ec867c7 100644 --- a/core/rawdb/freezer_test.go +++ b/core/rawdb/freezer_test.go @@ -277,7 +277,7 @@ func TestFreezerReadonlyValidate(t *testing.T) { // Re-openening as readonly should fail when validating // table lengths. - f, err = NewFreezer(dir, "", true, 2049, tables) + _, err = NewFreezer(dir, "", true, 2049, tables) if err == nil { t.Fatal("readonly freezer should fail with differing table lengths") } diff --git a/core/rawdb/freezer_utils_test.go b/core/rawdb/freezer_utils_test.go index cc300cb614fa0..829cbfb4f332d 100644 --- a/core/rawdb/freezer_utils_test.go +++ b/core/rawdb/freezer_utils_test.go @@ -43,7 +43,7 @@ func TestCopyFrom(t *testing.T) { {"foo", "bar", 8, true}, } for _, c := range cases { - os.WriteFile(c.src, content, 0644) + os.WriteFile(c.src, content, 0600) if err := copyFrom(c.src, c.dest, c.offset, func(f *os.File) error { if !c.writePrefix { diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index 041c9f0449671..d5f751da3a13c 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -111,33 +111,6 @@ var ( preimageHitCounter = metrics.NewRegisteredCounter("db/preimage/hits", nil) ) -const ( - // freezerHeaderTable indicates the name of the freezer header table. - freezerHeaderTable = "headers" - - // freezerHashTable indicates the name of the freezer canonical hash table. - freezerHashTable = "hashes" - - // freezerBodiesTable indicates the name of the freezer block body table. - freezerBodiesTable = "bodies" - - // freezerReceiptTable indicates the name of the freezer receipts table. - freezerReceiptTable = "receipts" - - // freezerDifficultyTable indicates the name of the freezer total difficulty table. - freezerDifficultyTable = "diffs" -) - -// FreezerNoSnappy configures whether compression is disabled for the ancient-tables. -// Hashes and difficulties don't compress well. -var FreezerNoSnappy = map[string]bool{ - freezerHeaderTable: false, - freezerHashTable: true, - freezerBodiesTable: false, - freezerReceiptTable: false, - freezerDifficultyTable: true, -} - // LegacyTxLookupEntry is the legacy TxLookupEntry definition with some unnecessary // fields. type LegacyTxLookupEntry struct { @@ -247,7 +220,7 @@ func configKey(hash common.Hash) []byte { return append(configPrefix, hash.Bytes()...) } -// genesisKey = genesisPrefix + hash -func genesisKey(hash common.Hash) []byte { +// genesisStateSpecKey = genesisPrefix + hash +func genesisStateSpecKey(hash common.Hash) []byte { return append(genesisPrefix, hash.Bytes()...) } diff --git a/core/rlp_test.go b/core/rlp_test.go index bf5a934ce5519..a2fb4937f8bbd 100644 --- a/core/rlp_test.go +++ b/core/rlp_test.go @@ -23,7 +23,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus/ethash" - "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/params" @@ -33,10 +32,9 @@ import ( func getBlock(transactions int, uncles int, dataSize int) *types.Block { var ( - aa = common.HexToAddress("0x000000000000000000000000000000000000aaaa") - // Generate a canonical chain to act as the main dataset + aa = common.HexToAddress("0x000000000000000000000000000000000000aaaa") engine = ethash.NewFaker() - db = rawdb.NewMemoryDatabase() + // A sender who makes transactions, has some funds key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") address = crypto.PubkeyToAddress(key.PublicKey) @@ -45,11 +43,9 @@ func getBlock(transactions int, uncles int, dataSize int) *types.Block { Config: params.TestChainConfig, Alloc: GenesisAlloc{address: {Balance: funds}}, } - genesis = gspec.MustCommit(db) ) - // We need to generate as many blocks +1 as uncles - blocks, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, uncles+1, + _, blocks, _ := GenerateChainWithGenesis(gspec, engine, uncles+1, func(n int, b *BlockGen) { if n == uncles { // Add transactions and stuff on the last block diff --git a/core/state/database.go b/core/state/database.go index bbcd2358e5b8e..5e3d9a9d388a2 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -43,7 +43,7 @@ type Database interface { OpenTrie(root common.Hash) (Trie, error) // OpenStorageTrie opens the storage trie of an account. - OpenStorageTrie(addrHash, root common.Hash) (Trie, error) + OpenStorageTrie(stateRoot common.Hash, addrHash, root common.Hash) (Trie, error) // CopyTrie returns an independent copy of the given trie. CopyTrie(Trie) Trie @@ -54,6 +54,9 @@ type Database interface { // ContractCodeSize retrieves a particular contracts code's size. ContractCodeSize(addrHash, codeHash common.Hash) (int, error) + // DiskDB returns the underlying key-value disk database. + DiskDB() ethdb.KeyValueStore + // TrieDB retrieves the low level trie database used for data storage. TrieDB() *trie.Database } @@ -63,7 +66,7 @@ type Trie interface { // GetKey returns the sha3 preimage of a hashed key that was previously used // to store a value. // - // TODO(fjl): remove this when SecureTrie is removed + // TODO(fjl): remove this when StateTrie is removed GetKey([]byte) []byte // TryGet returns the value for key stored in the trie. The value bytes must @@ -71,8 +74,8 @@ type Trie interface { // trie.MissingNodeError is returned. TryGet(key []byte) ([]byte, error) - // TryUpdateAccount abstract an account write in the trie. - TryUpdateAccount(key []byte, account *types.StateAccount) error + // TryGetAccount abstract an account read from the trie. + TryGetAccount(key []byte) (*types.StateAccount, error) // TryUpdate associates key with value in the trie. If value has length zero, any // existing value is deleted from the trie. The value bytes must not be modified @@ -80,17 +83,27 @@ type Trie interface { // database, a trie.MissingNodeError is returned. TryUpdate(key, value []byte) error + // TryUpdateAccount abstract an account write to the trie. + TryUpdateAccount(key []byte, account *types.StateAccount) error + // TryDelete removes any existing value for key from the trie. If a node was not // found in the database, a trie.MissingNodeError is returned. TryDelete(key []byte) error + // TryDeleteAccount abstracts an account deletion from the trie. + TryDeleteAccount(key []byte) error + // Hash returns the root hash of the trie. It does not write to the database and // can be used even if the trie doesn't have one. Hash() common.Hash - // Commit writes all nodes to the trie's memory database, tracking the internal - // and external (for account tries) references. - Commit(onleaf trie.LeafCallback) (common.Hash, int, error) + // Commit collects all dirty nodes in the trie and replace them with the + // corresponding node hash. All collected nodes(including dirty leaves if + // collectLeaf is true) will be encapsulated into a nodeset for return. + // The returned nodeset can be nil if the trie is clean(nothing to commit). + // Once the trie is committed, it's not usable anymore. A new trie must + // be created with new root and updated trie database for following usage + Commit(collectLeaf bool) (common.Hash, *trie.NodeSet, error) // NodeIterator returns an iterator that returns nodes of the trie. Iteration // starts at the key after the given start key. @@ -120,6 +133,7 @@ func NewDatabaseWithConfig(db ethdb.Database, config *trie.Config) Database { csc, _ := lru.New(codeSizeCacheSize) return &cachingDB{ db: trie.NewDatabaseWithConfig(db, config), + disk: db, codeSizeCache: csc, codeCache: fastcache.New(codeCacheSize), } @@ -127,13 +141,14 @@ func NewDatabaseWithConfig(db ethdb.Database, config *trie.Config) Database { type cachingDB struct { db *trie.Database + disk ethdb.KeyValueStore codeSizeCache *lru.Cache codeCache *fastcache.Cache } // OpenTrie opens the main account trie at a specific root hash. func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) { - tr, err := trie.NewSecure(root, db.db) + tr, err := trie.NewStateTrie(trie.StateTrieID(root), db.db) if err != nil { return nil, err } @@ -141,8 +156,8 @@ func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) { } // OpenStorageTrie opens the storage trie of an account. -func (db *cachingDB) OpenStorageTrie(addrHash, root common.Hash) (Trie, error) { - tr, err := trie.NewSecure(root, db.db) +func (db *cachingDB) OpenStorageTrie(stateRoot common.Hash, addrHash, root common.Hash) (Trie, error) { + tr, err := trie.NewStateTrie(trie.StorageTrieID(stateRoot, addrHash, root), db.db) if err != nil { return nil, err } @@ -152,7 +167,7 @@ func (db *cachingDB) OpenStorageTrie(addrHash, root common.Hash) (Trie, error) { // CopyTrie returns an independent copy of the given trie. func (db *cachingDB) CopyTrie(t Trie) Trie { switch t := t.(type) { - case *trie.SecureTrie: + case *trie.StateTrie: return t.Copy() default: panic(fmt.Errorf("unknown trie type %T", t)) @@ -164,7 +179,7 @@ func (db *cachingDB) ContractCode(addrHash, codeHash common.Hash) ([]byte, error if code := db.codeCache.Get(nil, codeHash.Bytes()); len(code) > 0 { return code, nil } - code := rawdb.ReadCode(db.db.DiskDB(), codeHash) + code := rawdb.ReadCode(db.disk, codeHash) if len(code) > 0 { db.codeCache.Set(codeHash.Bytes(), code) db.codeSizeCache.Add(codeHash, len(code)) @@ -180,7 +195,7 @@ func (db *cachingDB) ContractCodeWithPrefix(addrHash, codeHash common.Hash) ([]b if code := db.codeCache.Get(nil, codeHash.Bytes()); len(code) > 0 { return code, nil } - code := rawdb.ReadCodeWithPrefix(db.db.DiskDB(), codeHash) + code := rawdb.ReadCodeWithPrefix(db.disk, codeHash) if len(code) > 0 { db.codeCache.Set(codeHash.Bytes(), code) db.codeSizeCache.Add(codeHash, len(code)) @@ -198,6 +213,11 @@ func (db *cachingDB) ContractCodeSize(addrHash, codeHash common.Hash) (int, erro return len(code), err } +// DiskDB returns the underlying key-value disk database. +func (db *cachingDB) DiskDB() ethdb.KeyValueStore { + return db.disk +} + // TrieDB retrieves any intermediate trie-node caching layer. func (db *cachingDB) TrieDB() *trie.Database { return db.db diff --git a/core/state/iterator.go b/core/state/iterator.go index 611df52431eb3..ba7efd4653b39 100644 --- a/core/state/iterator.go +++ b/core/state/iterator.go @@ -109,7 +109,7 @@ func (it *NodeIterator) step() error { if err := rlp.Decode(bytes.NewReader(it.stateIt.LeafBlob()), &account); err != nil { return err } - dataTrie, err := it.state.db.OpenStorageTrie(common.BytesToHash(it.stateIt.LeafKey()), account.Root) + dataTrie, err := it.state.db.OpenStorageTrie(it.state.originalRoot, common.BytesToHash(it.stateIt.LeafKey()), account.Root) if err != nil { return err } diff --git a/core/state/iterator_test.go b/core/state/iterator_test.go index d1afe9ca3eb72..f9337512647ad 100644 --- a/core/state/iterator_test.go +++ b/core/state/iterator_test.go @@ -21,7 +21,6 @@ import ( "testing" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/ethdb" ) // Tests that the node iterator indeed walks over the entire database contents. @@ -55,7 +54,7 @@ func TestNodeIteratorCoverage(t *testing.T) { t.Errorf("state entry not reported %x", hash) } } - it := db.TrieDB().DiskDB().(ethdb.Database).NewIterator(nil, nil) + it := db.DiskDB().NewIterator(nil, nil) for it.Next() { key := it.Key() if bytes.HasPrefix(key, []byte("secure-key-")) { diff --git a/core/state/metrics.go b/core/state/metrics.go index 7b40ff37aff0e..e702ef3a81a60 100644 --- a/core/state/metrics.go +++ b/core/state/metrics.go @@ -19,10 +19,12 @@ package state import "github.com/ethereum/go-ethereum/metrics" var ( - accountUpdatedMeter = metrics.NewRegisteredMeter("state/update/account", nil) - storageUpdatedMeter = metrics.NewRegisteredMeter("state/update/storage", nil) - accountDeletedMeter = metrics.NewRegisteredMeter("state/delete/account", nil) - storageDeletedMeter = metrics.NewRegisteredMeter("state/delete/storage", nil) - accountCommittedMeter = metrics.NewRegisteredMeter("state/commit/account", nil) - storageCommittedMeter = metrics.NewRegisteredMeter("state/commit/storage", nil) + accountUpdatedMeter = metrics.NewRegisteredMeter("state/update/account", nil) + storageUpdatedMeter = metrics.NewRegisteredMeter("state/update/storage", nil) + accountDeletedMeter = metrics.NewRegisteredMeter("state/delete/account", nil) + storageDeletedMeter = metrics.NewRegisteredMeter("state/delete/storage", nil) + accountTrieUpdatedMeter = metrics.NewRegisteredMeter("state/update/accountnodes", nil) + storageTriesUpdatedMeter = metrics.NewRegisteredMeter("state/update/storagenodes", nil) + accountTrieDeletedMeter = metrics.NewRegisteredMeter("state/delete/accountnodes", nil) + storageTriesDeletedMeter = metrics.NewRegisteredMeter("state/delete/storagenodes", nil) ) diff --git a/core/state/pruner/bloom.go b/core/state/pruner/bloom.go index 29bc4e7314efa..72315db720f19 100644 --- a/core/state/pruner/bloom.go +++ b/core/state/pruner/bloom.go @@ -39,7 +39,7 @@ func (f stateBloomHasher) BlockSize() int { panic("not implem func (f stateBloomHasher) Size() int { return 8 } func (f stateBloomHasher) Sum64() uint64 { return binary.BigEndian.Uint64(f) } -// stateBloom is a bloom filter used during the state convesion(snapshot->state). +// stateBloom is a bloom filter used during the state conversion(snapshot->state). // The keys of all generated entries will be recorded here so that in the pruning // stage the entries belong to the specific version can be avoided for deletion. // @@ -100,7 +100,7 @@ func (bloom *stateBloom) Commit(filename, tempname string) error { } f.Close() - // Move the teporary file into it's final location + // Move the temporary file into it's final location return os.Rename(tempname, filename) } diff --git a/core/state/pruner/pruner.go b/core/state/pruner/pruner.go index a121839bd099f..214699208471c 100644 --- a/core/state/pruner/pruner.go +++ b/core/state/pruner/pruner.go @@ -63,52 +63,63 @@ var ( emptyCode = crypto.Keccak256(nil) ) +// Config includes all the configurations for pruning. +type Config struct { + Datadir string // The directory of the state database + Cachedir string // The directory of state clean cache + BloomSize uint64 // The Megabytes of memory allocated to bloom-filter +} + // Pruner is an offline tool to prune the stale state with the // help of the snapshot. The workflow of pruner is very simple: // -// - iterate the snapshot, reconstruct the relevant state -// - iterate the database, delete all other state entries which -// don't belong to the target state and the genesis state +// - iterate the snapshot, reconstruct the relevant state +// - iterate the database, delete all other state entries which +// don't belong to the target state and the genesis state // // It can take several hours(around 2 hours for mainnet) to finish // the whole pruning work. It's recommended to run this offline tool // periodically in order to release the disk usage and improve the // disk read performance to some extent. type Pruner struct { - db ethdb.Database - stateBloom *stateBloom - datadir string - trieCachePath string - headHeader *types.Header - snaptree *snapshot.Tree + config Config + chainHeader *types.Header + db ethdb.Database + stateBloom *stateBloom + snaptree *snapshot.Tree } // NewPruner creates the pruner instance. -func NewPruner(db ethdb.Database, datadir, trieCachePath string, bloomSize uint64) (*Pruner, error) { +func NewPruner(db ethdb.Database, config Config) (*Pruner, error) { headBlock := rawdb.ReadHeadBlock(db) if headBlock == nil { - return nil, errors.New("Failed to load head block") + return nil, errors.New("failed to load head block") } - snaptree, err := snapshot.New(db, trie.NewDatabase(db), 256, headBlock.Root(), false, false, false) + snapconfig := snapshot.Config{ + CacheSize: 256, + Recovery: false, + NoBuild: true, + AsyncBuild: false, + } + snaptree, err := snapshot.New(snapconfig, db, trie.NewDatabase(db), headBlock.Root()) if err != nil { return nil, err // The relevant snapshot(s) might not exist } // Sanitize the bloom filter size if it's too small. - if bloomSize < 256 { - log.Warn("Sanitizing bloomfilter size", "provided(MB)", bloomSize, "updated(MB)", 256) - bloomSize = 256 + if config.BloomSize < 256 { + log.Warn("Sanitizing bloomfilter size", "provided(MB)", config.BloomSize, "updated(MB)", 256) + config.BloomSize = 256 } - stateBloom, err := newStateBloomWithSize(bloomSize) + stateBloom, err := newStateBloomWithSize(config.BloomSize) if err != nil { return nil, err } return &Pruner{ - db: db, - stateBloom: stateBloom, - datadir: datadir, - trieCachePath: trieCachePath, - headHeader: headBlock.Header(), - snaptree: snaptree, + config: config, + chainHeader: headBlock.Header(), + db: db, + stateBloom: stateBloom, + snaptree: snaptree, }, nil } @@ -236,12 +247,12 @@ func (p *Pruner) Prune(root common.Hash) error { // reuse it for pruning instead of generating a new one. It's // mandatory because a part of state may already be deleted, // the recovery procedure is necessary. - _, stateBloomRoot, err := findBloomFilter(p.datadir) + _, stateBloomRoot, err := findBloomFilter(p.config.Datadir) if err != nil { return err } if stateBloomRoot != (common.Hash{}) { - return RecoverPruning(p.datadir, p.db, p.trieCachePath) + return RecoverPruning(p.config.Datadir, p.db, p.config.Cachedir) } // If the target state root is not specified, use the HEAD-127 as the // target. The reason for picking it is: @@ -252,7 +263,7 @@ func (p *Pruner) Prune(root common.Hash) error { // Retrieve all snapshot layers from the current HEAD. // In theory there are 128 difflayers + 1 disk layer present, // so 128 diff layers are expected to be returned. - layers = p.snaptree.Snapshots(p.headHeader.Root, 128, true) + layers = p.snaptree.Snapshots(p.chainHeader.Root, 128, true) if len(layers) != 128 { // Reject if the accumulated diff layers are less than 128. It // means in most of normal cases, there is no associated state @@ -294,7 +305,7 @@ func (p *Pruner) Prune(root common.Hash) error { } } else { if len(layers) > 0 { - log.Info("Selecting bottom-most difflayer as the pruning target", "root", root, "height", p.headHeader.Number.Uint64()-127) + log.Info("Selecting bottom-most difflayer as the pruning target", "root", root, "height", p.chainHeader.Number.Uint64()-127) } else { log.Info("Selecting user-specified state as the pruning target", "root", root) } @@ -303,7 +314,7 @@ func (p *Pruner) Prune(root common.Hash) error { // It's necessary otherwise in the next restart we will hit the // deleted state root in the "clean cache" so that the incomplete // state is picked for usage. - deleteCleanTrieCache(p.trieCachePath) + deleteCleanTrieCache(p.config.Cachedir) // All the state roots of the middle layer should be forcibly pruned, // otherwise the dangling state will be left. @@ -325,7 +336,7 @@ func (p *Pruner) Prune(root common.Hash) error { if err := extractGenesis(p.db, p.stateBloom); err != nil { return err } - filterName := bloomFilterName(p.datadir, root) + filterName := bloomFilterName(p.config.Datadir, root) log.Info("Writing state bloom to disk", "name", filterName) if err := p.stateBloom.Commit(filterName, filterName+stateBloomFileTempSuffix); err != nil { @@ -362,7 +373,13 @@ func RecoverPruning(datadir string, db ethdb.Database, trieCachePath string) err // - The state HEAD is rewound already because of multiple incomplete `prune-state` // In this case, even the state HEAD is not exactly matched with snapshot, it // still feasible to recover the pruning correctly. - snaptree, err := snapshot.New(db, trie.NewDatabase(db), 256, headBlock.Root(), false, false, true) + snapconfig := snapshot.Config{ + CacheSize: 256, + Recovery: true, + NoBuild: true, + AsyncBuild: false, + } + snaptree, err := snapshot.New(snapconfig, db, trie.NewDatabase(db), headBlock.Root()) if err != nil { return err // The relevant snapshot(s) might not exist } @@ -410,7 +427,7 @@ func extractGenesis(db ethdb.Database, stateBloom *stateBloom) error { if genesis == nil { return errors.New("missing genesis block") } - t, err := trie.NewSecure(genesis.Root(), trie.NewDatabase(db)) + t, err := trie.NewStateTrie(trie.StateTrieID(genesis.Root()), trie.NewDatabase(db)) if err != nil { return err } @@ -430,7 +447,8 @@ func extractGenesis(db ethdb.Database, stateBloom *stateBloom) error { return err } if acc.Root != emptyRoot { - storageTrie, err := trie.NewSecure(acc.Root, trie.NewDatabase(db)) + id := trie.StorageTrieID(genesis.Root(), common.BytesToHash(accIter.LeafKey()), acc.Root) + storageTrie, err := trie.NewStateTrie(id, trie.NewDatabase(db)) if err != nil { return err } diff --git a/core/state/snapshot/conversion.go b/core/state/snapshot/conversion.go index f70cbf1e686b1..0f3934cb423bb 100644 --- a/core/state/snapshot/conversion.go +++ b/core/state/snapshot/conversion.go @@ -43,7 +43,7 @@ type trieKV struct { type ( // trieGeneratorFn is the interface of trie generation which can // be implemented by different trie algorithm. - trieGeneratorFn func(db ethdb.KeyValueWriter, in chan (trieKV), out chan (common.Hash)) + trieGeneratorFn func(db ethdb.KeyValueWriter, owner common.Hash, in chan (trieKV), out chan (common.Hash)) // leafCallbackFn is the callback invoked at the leaves of the trie, // returns the subtrie root with the specified subtrie identifier. @@ -253,7 +253,7 @@ func generateTrieRoot(db ethdb.KeyValueWriter, it Iterator, account common.Hash, wg.Add(1) go func() { defer wg.Done() - generatorFn(db, in, out) + generatorFn(db, account, in, out) }() // Spin up a go-routine for progress logging if report && stats != nil { @@ -360,8 +360,8 @@ func generateTrieRoot(db ethdb.KeyValueWriter, it Iterator, account common.Hash, return stop(nil) } -func stackTrieGenerate(db ethdb.KeyValueWriter, in chan trieKV, out chan common.Hash) { - t := trie.NewStackTrie(db) +func stackTrieGenerate(db ethdb.KeyValueWriter, owner common.Hash, in chan trieKV, out chan common.Hash) { + t := trie.NewStackTrieWithOwner(db, owner) for leaf := range in { t.TryUpdate(leaf.key[:], leaf.value) } diff --git a/core/state/snapshot/difflayer.go b/core/state/snapshot/difflayer.go index 822c91f15cb8d..f916a020e7bcf 100644 --- a/core/state/snapshot/difflayer.go +++ b/core/state/snapshot/difflayer.go @@ -68,7 +68,7 @@ var ( bloomFuncs = math.Round((bloomSize / float64(aggregatorItemLimit)) * math.Log(2)) // the bloom offsets are runtime constants which determines which part of the - // the account/storage hash the hasher functions looks at, to determine the + // account/storage hash the hasher functions looks at, to determine the // bloom key for an account/slot. This is randomized at init(), so that the // global population of nodes do not all display the exact same behaviour with // regards to bloom content diff --git a/core/state/snapshot/difflayer_test.go b/core/state/snapshot/difflayer_test.go index e15c1d5049b03..59db920481b01 100644 --- a/core/state/snapshot/difflayer_test.go +++ b/core/state/snapshot/difflayer_test.go @@ -332,7 +332,6 @@ func BenchmarkFlatten(b *testing.B) { value := make([]byte, 32) rand.Read(value) accStorage[randomHash()] = value - } storage[accountKey] = accStorage } @@ -382,7 +381,6 @@ func BenchmarkJournal(b *testing.B) { value := make([]byte, 32) rand.Read(value) accStorage[randomHash()] = value - } storage[accountKey] = accStorage } diff --git a/core/state/snapshot/disklayer_test.go b/core/state/snapshot/disklayer_test.go index b078951c72aa0..f95b79851598e 100644 --- a/core/state/snapshot/disklayer_test.go +++ b/core/state/snapshot/disklayer_test.go @@ -23,7 +23,6 @@ import ( "github.com/VictoriaMetrics/fastcache" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/ethdb/leveldb" "github.com/ethereum/go-ethereum/ethdb/memorydb" "github.com/ethereum/go-ethereum/rlp" ) @@ -515,11 +514,7 @@ func TestDiskMidAccountPartialMerge(t *testing.T) { // TestDiskSeek tests that seek-operations work on the disk layer func TestDiskSeek(t *testing.T) { // Create some accounts in the disk layer - diskdb, err := leveldb.New(t.TempDir(), 256, 0, "", false) - if err != nil { - t.Fatal(err) - } - db := rawdb.NewDatabase(diskdb) + db := rawdb.NewMemoryDatabase() defer db.Close() // Fill even keys [0,2,4...] diff --git a/core/state/snapshot/generate.go b/core/state/snapshot/generate.go index 769989aec21c0..8589aa784f670 100644 --- a/core/state/snapshot/generate.go +++ b/core/state/snapshot/generate.go @@ -166,7 +166,7 @@ func (result *proofResult) forEach(callback func(key []byte, val []byte) error) // // The proof result will be returned if the range proving is finished, otherwise // the error will be returned to abort the entire procedure. -func (dl *diskLayer) proveRange(ctx *generatorContext, root common.Hash, prefix []byte, kind string, origin []byte, max int, valueConvertFn func([]byte) ([]byte, error)) (*proofResult, error) { +func (dl *diskLayer) proveRange(ctx *generatorContext, trieId *trie.ID, prefix []byte, kind string, origin []byte, max int, valueConvertFn func([]byte) ([]byte, error)) (*proofResult, error) { var ( keys [][]byte vals [][]byte @@ -233,6 +233,7 @@ func (dl *diskLayer) proveRange(ctx *generatorContext, root common.Hash, prefix }(time.Now()) // The snap state is exhausted, pass the entire key/val set for verification + root := trieId.Root if origin == nil && !diskMore { stackTr := trie.NewStackTrie(nil) for i, key := range keys { @@ -248,7 +249,7 @@ func (dl *diskLayer) proveRange(ctx *generatorContext, root common.Hash, prefix return &proofResult{keys: keys, vals: vals}, nil } // Snap state is chunked, generate edge proofs for verification. - tr, err := trie.New(root, dl.triedb) + tr, err := trie.New(trieId, dl.triedb) if err != nil { ctx.stats.Log("Trie missing, state snapshotting paused", dl.root, dl.genMarker) return nil, errMissingTrie @@ -313,9 +314,9 @@ type onStateCallback func(key []byte, val []byte, write bool, delete bool) error // generateRange generates the state segment with particular prefix. Generation can // either verify the correctness of existing state through range-proof and skip // generation, or iterate trie to regenerate state on demand. -func (dl *diskLayer) generateRange(ctx *generatorContext, root common.Hash, prefix []byte, kind string, origin []byte, max int, onState onStateCallback, valueConvertFn func([]byte) ([]byte, error)) (bool, []byte, error) { +func (dl *diskLayer) generateRange(ctx *generatorContext, trieId *trie.ID, prefix []byte, kind string, origin []byte, max int, onState onStateCallback, valueConvertFn func([]byte) ([]byte, error)) (bool, []byte, error) { // Use range prover to check the validity of the flat state in the range - result, err := dl.proveRange(ctx, root, prefix, kind, origin, max, valueConvertFn) + result, err := dl.proveRange(ctx, trieId, prefix, kind, origin, max, valueConvertFn) if err != nil { return false, nil, err } @@ -363,18 +364,21 @@ func (dl *diskLayer) generateRange(ctx *generatorContext, root common.Hash, pref if len(result.keys) > 0 { snapNodeCache = memorydb.New() snapTrieDb := trie.NewDatabase(snapNodeCache) - snapTrie, _ := trie.New(common.Hash{}, snapTrieDb) + snapTrie := trie.NewEmpty(snapTrieDb) for i, key := range result.keys { snapTrie.Update(key, result.vals[i]) } - root, _, _ := snapTrie.Commit(nil) + root, nodes, _ := snapTrie.Commit(false) + if nodes != nil { + snapTrieDb.Update(trie.NewWithNodeSet(nodes)) + } snapTrieDb.Commit(root, false, nil) } // Construct the trie for state iteration, reuse the trie // if it's already opened with some nodes resolved. tr := result.tr if tr == nil { - tr, err = trie.New(root, dl.triedb) + tr, err = trie.New(trieId, dl.triedb) if err != nil { ctx.stats.Log("Trie missing, state snapshotting paused", dl.root, dl.genMarker) return false, nil, errMissingTrie @@ -457,7 +461,7 @@ func (dl *diskLayer) generateRange(ctx *generatorContext, root common.Hash, pref } else { snapAccountTrieReadCounter.Inc((time.Since(start) - internal).Nanoseconds()) } - logger.Debug("Regenerated state range", "root", root, "last", hexutil.Encode(last), + logger.Debug("Regenerated state range", "root", trieId.Root, "last", hexutil.Encode(last), "count", count, "created", created, "updated", updated, "untouched", untouched, "deleted", deleted) // If there are either more trie items, or there are more snap items @@ -508,7 +512,7 @@ func (dl *diskLayer) checkAndFlush(ctx *generatorContext, current []byte) error // generateStorages generates the missing storage slots of the specific contract. // It's supposed to restart the generation from the given origin position. -func generateStorages(ctx *generatorContext, dl *diskLayer, account common.Hash, storageRoot common.Hash, storeMarker []byte) error { +func generateStorages(ctx *generatorContext, dl *diskLayer, stateRoot common.Hash, account common.Hash, storageRoot common.Hash, storeMarker []byte) error { onStorage := func(key []byte, val []byte, write bool, delete bool) error { defer func(start time.Time) { snapStorageWriteCounter.Inc(time.Since(start).Nanoseconds()) @@ -537,7 +541,8 @@ func generateStorages(ctx *generatorContext, dl *diskLayer, account common.Hash, // Loop for re-generating the missing storage slots. var origin = common.CopyBytes(storeMarker) for { - exhausted, last, err := dl.generateRange(ctx, storageRoot, append(rawdb.SnapshotStoragePrefix, account.Bytes()...), snapStorage, origin, storageCheckRange, onStorage, nil) + id := trie.StorageTrieID(stateRoot, account, storageRoot) + exhausted, last, err := dl.generateRange(ctx, id, append(rawdb.SnapshotStoragePrefix, account.Bytes()...), snapStorage, origin, storageCheckRange, onStorage, nil) if err != nil { return err // The procedure it aborted, either by external signal or internal error. } @@ -621,7 +626,7 @@ func generateAccounts(ctx *generatorContext, dl *diskLayer, accMarker []byte) er if accMarker != nil && bytes.Equal(account[:], accMarker) && len(dl.genMarker) > common.HashLength { storeMarker = dl.genMarker[common.HashLength:] } - if err := generateStorages(ctx, dl, account, acc.Root, storeMarker); err != nil { + if err := generateStorages(ctx, dl, dl.root, account, acc.Root, storeMarker); err != nil { return err } } @@ -637,7 +642,8 @@ func generateAccounts(ctx *generatorContext, dl *diskLayer, accMarker []byte) er } origin := common.CopyBytes(accMarker) for { - exhausted, last, err := dl.generateRange(ctx, dl.root, rawdb.SnapshotAccountPrefix, snapAccount, origin, accountRange, onAccount, FullAccountRLP) + id := trie.StateTrieID(dl.root) + exhausted, last, err := dl.generateRange(ctx, id, rawdb.SnapshotAccountPrefix, snapAccount, origin, accountRange, onAccount, FullAccountRLP) if err != nil { return err // The procedure it aborted, either by external signal or internal error. } diff --git a/core/state/snapshot/generate_test.go b/core/state/snapshot/generate_test.go index 94caed08ad7a7..784d76859e448 100644 --- a/core/state/snapshot/generate_test.go +++ b/core/state/snapshot/generate_test.go @@ -26,47 +26,40 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/ethdb/memorydb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" "golang.org/x/crypto/sha3" ) +func hashData(input []byte) common.Hash { + var hasher = sha3.NewLegacyKeccak256() + var hash common.Hash + hasher.Reset() + hasher.Write(input) + hasher.Sum(hash[:0]) + return hash +} + // Tests that snapshot generation from an empty database. func TestGeneration(t *testing.T) { // We can't use statedb to make a test trie (circular dependency), so make // a fake one manually. We're going with a small account trie of 3 accounts, // two of which also has the same 3-slot storage trie attached. - var ( - diskdb = memorydb.New() - triedb = trie.NewDatabase(diskdb) - ) - stTrie, _ := trie.NewSecure(common.Hash{}, triedb) - stTrie.Update([]byte("key-1"), []byte("val-1")) // 0x1314700b81afc49f94db3623ef1df38f3ed18b73a1b7ea2f6c095118cf6118a0 - stTrie.Update([]byte("key-2"), []byte("val-2")) // 0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371 - stTrie.Update([]byte("key-3"), []byte("val-3")) // 0x51c71a47af0695957647fb68766d0becee77e953df17c29b3c2f25436f055c78 - stTrie.Commit(nil) // Root: 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67 - - accTrie, _ := trie.NewSecure(common.Hash{}, triedb) - acc := &Account{Balance: big.NewInt(1), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()} - val, _ := rlp.EncodeToBytes(acc) - accTrie.Update([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e + var helper = newHelper() + stRoot := helper.makeStorageTrie(common.Hash{}, common.Hash{}, []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, false) - acc = &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()} - val, _ = rlp.EncodeToBytes(acc) - accTrie.Update([]byte("acc-2"), val) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7 + helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) + helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) + helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: emptyCode.Bytes()}) - acc = &Account{Balance: big.NewInt(3), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()} - val, _ = rlp.EncodeToBytes(acc) - accTrie.Update([]byte("acc-3"), val) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2 - root, _, _ := accTrie.Commit(nil) // Root: 0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd - triedb.Commit(root, false, nil) + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) + root, snap := helper.CommitAndGenerate() if have, want := root, common.HexToHash("0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd"); have != want { t.Fatalf("have %#x want %#x", have, want) } - snap := generateSnapshot(diskdb, triedb, 16, root) select { case <-snap.genPending: // Snapshot generation succeeded @@ -75,63 +68,34 @@ func TestGeneration(t *testing.T) { t.Errorf("Snapshot generation failed") } checkSnapRoot(t, snap, root) + // Signal abortion to the generator and wait for it to tear down stop := make(chan *generatorStats) snap.genAbort <- stop <-stop } -func hashData(input []byte) common.Hash { - var hasher = sha3.NewLegacyKeccak256() - var hash common.Hash - hasher.Reset() - hasher.Write(input) - hasher.Sum(hash[:0]) - return hash -} - // Tests that snapshot generation with existent flat state. func TestGenerateExistentState(t *testing.T) { // We can't use statedb to make a test trie (circular dependency), so make // a fake one manually. We're going with a small account trie of 3 accounts, // two of which also has the same 3-slot storage trie attached. - var ( - diskdb = memorydb.New() - triedb = trie.NewDatabase(diskdb) - ) - stTrie, _ := trie.NewSecure(common.Hash{}, triedb) - stTrie.Update([]byte("key-1"), []byte("val-1")) // 0x1314700b81afc49f94db3623ef1df38f3ed18b73a1b7ea2f6c095118cf6118a0 - stTrie.Update([]byte("key-2"), []byte("val-2")) // 0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371 - stTrie.Update([]byte("key-3"), []byte("val-3")) // 0x51c71a47af0695957647fb68766d0becee77e953df17c29b3c2f25436f055c78 - stTrie.Commit(nil) // Root: 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67 - - accTrie, _ := trie.NewSecure(common.Hash{}, triedb) - acc := &Account{Balance: big.NewInt(1), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()} - val, _ := rlp.EncodeToBytes(acc) - accTrie.Update([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e - rawdb.WriteAccountSnapshot(diskdb, hashData([]byte("acc-1")), val) - rawdb.WriteStorageSnapshot(diskdb, hashData([]byte("acc-1")), hashData([]byte("key-1")), []byte("val-1")) - rawdb.WriteStorageSnapshot(diskdb, hashData([]byte("acc-1")), hashData([]byte("key-2")), []byte("val-2")) - rawdb.WriteStorageSnapshot(diskdb, hashData([]byte("acc-1")), hashData([]byte("key-3")), []byte("val-3")) - - acc = &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()} - val, _ = rlp.EncodeToBytes(acc) - accTrie.Update([]byte("acc-2"), val) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7 - diskdb.Put(hashData([]byte("acc-2")).Bytes(), val) - rawdb.WriteAccountSnapshot(diskdb, hashData([]byte("acc-2")), val) - - acc = &Account{Balance: big.NewInt(3), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()} - val, _ = rlp.EncodeToBytes(acc) - accTrie.Update([]byte("acc-3"), val) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2 - rawdb.WriteAccountSnapshot(diskdb, hashData([]byte("acc-3")), val) - rawdb.WriteStorageSnapshot(diskdb, hashData([]byte("acc-3")), hashData([]byte("key-1")), []byte("val-1")) - rawdb.WriteStorageSnapshot(diskdb, hashData([]byte("acc-3")), hashData([]byte("key-2")), []byte("val-2")) - rawdb.WriteStorageSnapshot(diskdb, hashData([]byte("acc-3")), hashData([]byte("key-3")), []byte("val-3")) - - root, _, _ := accTrie.Commit(nil) // Root: 0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd - triedb.Commit(root, false, nil) - - snap := generateSnapshot(diskdb, triedb, 16, root) + var helper = newHelper() + + stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) + helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) + helper.addSnapAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) + helper.addSnapStorage("acc-1", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) + + helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) + helper.addSnapAccount("acc-2", &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) + + stRoot = helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) + helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: emptyCode.Bytes()}) + helper.addSnapAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: emptyCode.Bytes()}) + helper.addSnapStorage("acc-3", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) + + root, snap := helper.CommitAndGenerate() select { case <-snap.genPending: // Snapshot generation succeeded @@ -140,6 +104,7 @@ func TestGenerateExistentState(t *testing.T) { t.Errorf("Snapshot generation failed") } checkSnapRoot(t, snap, root) + // Signal abortion to the generator and wait for it to tear down stop := make(chan *generatorStats) snap.genAbort <- stop @@ -163,7 +128,6 @@ func checkSnapRoot(t *testing.T, snap *diskLayer, trieRoot common.Hash) { } return hash, nil }, newGenerateStats(), true) - if err != nil { t.Fatal(err) } @@ -171,24 +135,26 @@ func checkSnapRoot(t *testing.T, snap *diskLayer, trieRoot common.Hash) { t.Fatalf("snaproot: %#x != trieroot #%x", snapRoot, trieRoot) } if err := CheckDanglingStorage(snap.diskdb); err != nil { - t.Fatalf("Detected dangling storages %v", err) + t.Fatalf("Detected dangling storages: %v", err) } } type testHelper struct { - diskdb *memorydb.Database + diskdb ethdb.Database triedb *trie.Database - accTrie *trie.SecureTrie + accTrie *trie.StateTrie + nodes *trie.MergedNodeSet } func newHelper() *testHelper { - diskdb := memorydb.New() + diskdb := rawdb.NewMemoryDatabase() triedb := trie.NewDatabase(diskdb) - accTrie, _ := trie.NewSecure(common.Hash{}, triedb) + accTrie, _ := trie.NewStateTrie(trie.StateTrieID(common.Hash{}), triedb) return &testHelper{ diskdb: diskdb, triedb: triedb, accTrie: accTrie, + nodes: trie.NewMergedNodeSet(), } } @@ -215,18 +181,34 @@ func (t *testHelper) addSnapStorage(accKey string, keys []string, vals []string) } } -func (t *testHelper) makeStorageTrie(keys []string, vals []string) []byte { - stTrie, _ := trie.NewSecure(common.Hash{}, t.triedb) +func (t *testHelper) makeStorageTrie(stateRoot, owner common.Hash, keys []string, vals []string, commit bool) []byte { + id := trie.StorageTrieID(stateRoot, owner, common.Hash{}) + stTrie, _ := trie.NewStateTrie(id, t.triedb) for i, k := range keys { stTrie.Update([]byte(k), []byte(vals[i])) } - root, _, _ := stTrie.Commit(nil) + if !commit { + return stTrie.Hash().Bytes() + } + root, nodes, _ := stTrie.Commit(false) + if nodes != nil { + t.nodes.Merge(nodes) + } return root.Bytes() } -func (t *testHelper) Generate() (common.Hash, *diskLayer) { - root, _, _ := t.accTrie.Commit(nil) +func (t *testHelper) Commit() common.Hash { + root, nodes, _ := t.accTrie.Commit(true) + if nodes != nil { + t.nodes.Merge(nodes) + } + t.triedb.Update(t.nodes) t.triedb.Commit(root, false, nil) + return root +} + +func (t *testHelper) CommitAndGenerate() (common.Hash, *diskLayer) { + root := t.Commit() snap := generateSnapshot(t.diskdb, t.triedb, 16, root) return root, snap } @@ -239,36 +221,41 @@ func (t *testHelper) Generate() (common.Hash, *diskLayer) { // - miss in the beginning // - miss in the middle // - miss in the end +// // - the contract(non-empty storage) has wrong storage slots // - wrong slots in the beginning // - wrong slots in the middle // - wrong slots in the end +// // - the contract(non-empty storage) has extra storage slots // - extra slots in the beginning // - extra slots in the middle // - extra slots in the end func TestGenerateExistentStateWithWrongStorage(t *testing.T) { helper := newHelper() - stRoot := helper.makeStorageTrie([]string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) // Account one, empty root but non-empty database helper.addAccount("acc-1", &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) helper.addSnapStorage("acc-1", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) // Account two, non empty root but empty database + stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-2")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addAccount("acc-2", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) // Miss slots { // Account three, non empty root but misses slots in the beginning + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addAccount("acc-3", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) helper.addSnapStorage("acc-3", []string{"key-2", "key-3"}, []string{"val-2", "val-3"}) // Account four, non empty root but misses slots in the middle + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-4")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addAccount("acc-4", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) helper.addSnapStorage("acc-4", []string{"key-1", "key-3"}, []string{"val-1", "val-3"}) // Account five, non empty root but misses slots in the end + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-5")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addAccount("acc-5", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) helper.addSnapStorage("acc-5", []string{"key-1", "key-2"}, []string{"val-1", "val-2"}) } @@ -276,18 +263,22 @@ func TestGenerateExistentStateWithWrongStorage(t *testing.T) { // Wrong storage slots { // Account six, non empty root but wrong slots in the beginning + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-6")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addAccount("acc-6", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) helper.addSnapStorage("acc-6", []string{"key-1", "key-2", "key-3"}, []string{"badval-1", "val-2", "val-3"}) // Account seven, non empty root but wrong slots in the middle + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-7")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addAccount("acc-7", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) helper.addSnapStorage("acc-7", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "badval-2", "val-3"}) // Account eight, non empty root but wrong slots in the end + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-8")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addAccount("acc-8", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) helper.addSnapStorage("acc-8", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "badval-3"}) // Account 9, non empty root but rotated slots + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-9")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addAccount("acc-9", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) helper.addSnapStorage("acc-9", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-3", "val-2"}) } @@ -295,19 +286,22 @@ func TestGenerateExistentStateWithWrongStorage(t *testing.T) { // Extra storage slots { // Account 10, non empty root but extra slots in the beginning + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-10")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addAccount("acc-10", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) helper.addSnapStorage("acc-10", []string{"key-0", "key-1", "key-2", "key-3"}, []string{"val-0", "val-1", "val-2", "val-3"}) // Account 11, non empty root but extra slots in the middle + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-11")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addAccount("acc-11", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) helper.addSnapStorage("acc-11", []string{"key-1", "key-2", "key-2-1", "key-3"}, []string{"val-1", "val-2", "val-2-1", "val-3"}) // Account 12, non empty root but extra slots in the end + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-12")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addAccount("acc-12", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) helper.addSnapStorage("acc-12", []string{"key-1", "key-2", "key-3", "key-4"}, []string{"val-1", "val-2", "val-3", "val-4"}) } - root, snap := helper.Generate() + root, snap := helper.CommitAndGenerate() t.Logf("Root: %#x\n", root) // Root = 0x8746cce9fd9c658b2cfd639878ed6584b7a2b3e73bb40f607fcfa156002429a0 select { @@ -331,7 +325,12 @@ func TestGenerateExistentStateWithWrongStorage(t *testing.T) { // - extra accounts func TestGenerateExistentStateWithWrongAccounts(t *testing.T) { helper := newHelper() - stRoot := helper.makeStorageTrie([]string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) + + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-2")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-4")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) + stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-6")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) // Trie accounts [acc-1, acc-2, acc-3, acc-4, acc-6] // Extra accounts [acc-0, acc-5, acc-7] @@ -359,7 +358,7 @@ func TestGenerateExistentStateWithWrongAccounts(t *testing.T) { helper.addSnapAccount("acc-7", &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyRoot.Bytes()}) // after the end } - root, snap := helper.Generate() + root, snap := helper.CommitAndGenerate() t.Logf("Root: %#x\n", root) // Root = 0x825891472281463511e7ebcc7f109e4f9200c20fa384754e11fd605cd98464e8 select { @@ -383,29 +382,19 @@ func TestGenerateCorruptAccountTrie(t *testing.T) { // We can't use statedb to make a test trie (circular dependency), so make // a fake one manually. We're going with a small account trie of 3 accounts, // without any storage slots to keep the test smaller. - var ( - diskdb = memorydb.New() - triedb = trie.NewDatabase(diskdb) - ) - tr, _ := trie.NewSecure(common.Hash{}, triedb) - acc := &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()} - val, _ := rlp.EncodeToBytes(acc) - tr.Update([]byte("acc-1"), val) // 0xc7a30f39aff471c95d8a837497ad0e49b65be475cc0953540f80cfcdbdcd9074 + helper := newHelper() - acc = &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()} - val, _ = rlp.EncodeToBytes(acc) - tr.Update([]byte("acc-2"), val) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7 + helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) // 0xc7a30f39aff471c95d8a837497ad0e49b65be475cc0953540f80cfcdbdcd9074 + helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7 + helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) // 0x19ead688e907b0fab07176120dceec244a72aff2f0aa51e8b827584e378772f4 - acc = &Account{Balance: big.NewInt(3), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()} - val, _ = rlp.EncodeToBytes(acc) - tr.Update([]byte("acc-3"), val) // 0x19ead688e907b0fab07176120dceec244a72aff2f0aa51e8b827584e378772f4 - tr.Commit(nil) // Root: 0xa04693ea110a31037fb5ee814308a6f1d76bdab0b11676bdf4541d2de55ba978 + root := helper.Commit() // Root: 0xa04693ea110a31037fb5ee814308a6f1d76bdab0b11676bdf4541d2de55ba978 // Delete an account trie leaf and ensure the generator chokes - triedb.Commit(common.HexToHash("0xa04693ea110a31037fb5ee814308a6f1d76bdab0b11676bdf4541d2de55ba978"), false, nil) - diskdb.Delete(common.HexToHash("0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7").Bytes()) + helper.triedb.Commit(root, false, nil) + helper.diskdb.Delete(common.HexToHash("0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7").Bytes()) - snap := generateSnapshot(diskdb, triedb, 16, common.HexToHash("0xa04693ea110a31037fb5ee814308a6f1d76bdab0b11676bdf4541d2de55ba978")) + snap := generateSnapshot(helper.diskdb, helper.triedb, 16, root) select { case <-snap.genPending: // Snapshot generation succeeded @@ -427,45 +416,20 @@ func TestGenerateMissingStorageTrie(t *testing.T) { // We can't use statedb to make a test trie (circular dependency), so make // a fake one manually. We're going with a small account trie of 3 accounts, // two of which also has the same 3-slot storage trie attached. - var ( - diskdb = memorydb.New() - triedb = trie.NewDatabase(diskdb) - ) - stTrie, _ := trie.NewSecure(common.Hash{}, triedb) - stTrie.Update([]byte("key-1"), []byte("val-1")) // 0x1314700b81afc49f94db3623ef1df38f3ed18b73a1b7ea2f6c095118cf6118a0 - stTrie.Update([]byte("key-2"), []byte("val-2")) // 0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371 - stTrie.Update([]byte("key-3"), []byte("val-3")) // 0x51c71a47af0695957647fb68766d0becee77e953df17c29b3c2f25436f055c78 - stTrie.Commit(nil) // Root: 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67 - - accTrie, _ := trie.NewSecure(common.Hash{}, triedb) - acc := &Account{Balance: big.NewInt(1), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()} - val, _ := rlp.EncodeToBytes(acc) - accTrie.Update([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e - - acc = &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()} - val, _ = rlp.EncodeToBytes(acc) - accTrie.Update([]byte("acc-2"), val) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7 - - acc = &Account{Balance: big.NewInt(3), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()} - val, _ = rlp.EncodeToBytes(acc) - accTrie.Update([]byte("acc-3"), val) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2 - accTrie.Commit(nil) // Root: 0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd - - // We can only corrupt the disk database, so flush the tries out - triedb.Reference( - common.HexToHash("0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67"), - common.HexToHash("0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e"), - ) - triedb.Reference( - common.HexToHash("0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67"), - common.HexToHash("0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2"), - ) - triedb.Commit(common.HexToHash("0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd"), false, nil) + helper := newHelper() + + stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) // 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67 + helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e + helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7 + stRoot = helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) + helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: emptyCode.Bytes()}) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2 + + root := helper.Commit() // Delete a storage trie root and ensure the generator chokes - diskdb.Delete(common.HexToHash("0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67").Bytes()) + helper.diskdb.Delete(stRoot) - snap := generateSnapshot(diskdb, triedb, 16, common.HexToHash("0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd")) + snap := generateSnapshot(helper.diskdb, helper.triedb, 16, root) select { case <-snap.genPending: // Snapshot generation succeeded @@ -486,45 +450,20 @@ func TestGenerateCorruptStorageTrie(t *testing.T) { // We can't use statedb to make a test trie (circular dependency), so make // a fake one manually. We're going with a small account trie of 3 accounts, // two of which also has the same 3-slot storage trie attached. - var ( - diskdb = memorydb.New() - triedb = trie.NewDatabase(diskdb) - ) - stTrie, _ := trie.NewSecure(common.Hash{}, triedb) - stTrie.Update([]byte("key-1"), []byte("val-1")) // 0x1314700b81afc49f94db3623ef1df38f3ed18b73a1b7ea2f6c095118cf6118a0 - stTrie.Update([]byte("key-2"), []byte("val-2")) // 0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371 - stTrie.Update([]byte("key-3"), []byte("val-3")) // 0x51c71a47af0695957647fb68766d0becee77e953df17c29b3c2f25436f055c78 - stTrie.Commit(nil) // Root: 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67 - - accTrie, _ := trie.NewSecure(common.Hash{}, triedb) - acc := &Account{Balance: big.NewInt(1), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()} - val, _ := rlp.EncodeToBytes(acc) - accTrie.Update([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e - - acc = &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()} - val, _ = rlp.EncodeToBytes(acc) - accTrie.Update([]byte("acc-2"), val) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7 - - acc = &Account{Balance: big.NewInt(3), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()} - val, _ = rlp.EncodeToBytes(acc) - accTrie.Update([]byte("acc-3"), val) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2 - accTrie.Commit(nil) // Root: 0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd - - // We can only corrupt the disk database, so flush the tries out - triedb.Reference( - common.HexToHash("0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67"), - common.HexToHash("0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e"), - ) - triedb.Reference( - common.HexToHash("0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67"), - common.HexToHash("0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2"), - ) - triedb.Commit(common.HexToHash("0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd"), false, nil) + helper := newHelper() + + stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) // 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67 + helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e + helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7 + stRoot = helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) + helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: emptyCode.Bytes()}) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2 + + root := helper.Commit() // Delete a storage trie leaf and ensure the generator chokes - diskdb.Delete(common.HexToHash("0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371").Bytes()) + helper.diskdb.Delete(common.HexToHash("0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371").Bytes()) - snap := generateSnapshot(diskdb, triedb, 16, common.HexToHash("0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd")) + snap := generateSnapshot(helper.diskdb, helper.triedb, 16, root) select { case <-snap.genPending: // Snapshot generation succeeded @@ -539,56 +478,51 @@ func TestGenerateCorruptStorageTrie(t *testing.T) { <-stop } -func getStorageTrie(n int, triedb *trie.Database) *trie.SecureTrie { - stTrie, _ := trie.NewSecure(common.Hash{}, triedb) - for i := 0; i < n; i++ { - k := fmt.Sprintf("key-%d", i) - v := fmt.Sprintf("val-%d", i) - stTrie.Update([]byte(k), []byte(v)) - } - stTrie.Commit(nil) - return stTrie -} - // Tests that snapshot generation when an extra account with storage exists in the snap state. func TestGenerateWithExtraAccounts(t *testing.T) { - var ( - diskdb = memorydb.New() - triedb = trie.NewDatabase(diskdb) - stTrie = getStorageTrie(5, triedb) - ) - accTrie, _ := trie.NewSecure(common.Hash{}, triedb) - { // Account one in the trie - acc := &Account{Balance: big.NewInt(1), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()} + helper := newHelper() + { + // Account one in the trie + stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), + []string{"key-1", "key-2", "key-3", "key-4", "key-5"}, + []string{"val-1", "val-2", "val-3", "val-4", "val-5"}, + true, + ) + acc := &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()} val, _ := rlp.EncodeToBytes(acc) - accTrie.Update([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e + helper.accTrie.Update([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e + // Identical in the snap key := hashData([]byte("acc-1")) - rawdb.WriteAccountSnapshot(diskdb, key, val) - rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("key-1")), []byte("val-1")) - rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("key-2")), []byte("val-2")) - rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("key-3")), []byte("val-3")) - rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("key-4")), []byte("val-4")) - rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("key-5")), []byte("val-5")) - } - { // Account two exists only in the snapshot - acc := &Account{Balance: big.NewInt(1), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()} + rawdb.WriteAccountSnapshot(helper.diskdb, key, val) + rawdb.WriteStorageSnapshot(helper.diskdb, key, hashData([]byte("key-1")), []byte("val-1")) + rawdb.WriteStorageSnapshot(helper.diskdb, key, hashData([]byte("key-2")), []byte("val-2")) + rawdb.WriteStorageSnapshot(helper.diskdb, key, hashData([]byte("key-3")), []byte("val-3")) + rawdb.WriteStorageSnapshot(helper.diskdb, key, hashData([]byte("key-4")), []byte("val-4")) + rawdb.WriteStorageSnapshot(helper.diskdb, key, hashData([]byte("key-5")), []byte("val-5")) + } + { + // Account two exists only in the snapshot + stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-2")), + []string{"key-1", "key-2", "key-3", "key-4", "key-5"}, + []string{"val-1", "val-2", "val-3", "val-4", "val-5"}, + true, + ) + acc := &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()} val, _ := rlp.EncodeToBytes(acc) key := hashData([]byte("acc-2")) - rawdb.WriteAccountSnapshot(diskdb, key, val) - rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("b-key-1")), []byte("b-val-1")) - rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("b-key-2")), []byte("b-val-2")) - rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("b-key-3")), []byte("b-val-3")) - } - root, _, _ := accTrie.Commit(nil) - t.Logf("root: %x", root) - triedb.Commit(root, false, nil) + rawdb.WriteAccountSnapshot(helper.diskdb, key, val) + rawdb.WriteStorageSnapshot(helper.diskdb, key, hashData([]byte("b-key-1")), []byte("b-val-1")) + rawdb.WriteStorageSnapshot(helper.diskdb, key, hashData([]byte("b-key-2")), []byte("b-val-2")) + rawdb.WriteStorageSnapshot(helper.diskdb, key, hashData([]byte("b-key-3")), []byte("b-val-3")) + } + root := helper.Commit() + // To verify the test: If we now inspect the snap db, there should exist extraneous storage items - if data := rawdb.ReadStorageSnapshot(diskdb, hashData([]byte("acc-2")), hashData([]byte("b-key-1"))); data == nil { + if data := rawdb.ReadStorageSnapshot(helper.diskdb, hashData([]byte("acc-2")), hashData([]byte("b-key-1"))); data == nil { t.Fatalf("expected snap storage to exist") } - - snap := generateSnapshot(diskdb, triedb, 16, root) + snap := generateSnapshot(helper.diskdb, helper.triedb, 16, root) select { case <-snap.genPending: // Snapshot generation succeeded @@ -597,12 +531,13 @@ func TestGenerateWithExtraAccounts(t *testing.T) { t.Errorf("Snapshot generation failed") } checkSnapRoot(t, snap, root) + // Signal abortion to the generator and wait for it to tear down stop := make(chan *generatorStats) snap.genAbort <- stop <-stop // If we now inspect the snap db, there should exist no extraneous storage items - if data := rawdb.ReadStorageSnapshot(diskdb, hashData([]byte("acc-2")), hashData([]byte("b-key-1"))); data != nil { + if data := rawdb.ReadStorageSnapshot(helper.diskdb, hashData([]byte("acc-2")), hashData([]byte("b-key-1"))); data != nil { t.Fatalf("expected slot to be removed, got %v", string(data)) } } @@ -616,37 +551,36 @@ func TestGenerateWithManyExtraAccounts(t *testing.T) { if false { enableLogging() } - var ( - diskdb = memorydb.New() - triedb = trie.NewDatabase(diskdb) - stTrie = getStorageTrie(3, triedb) - ) - accTrie, _ := trie.NewSecure(common.Hash{}, triedb) - { // Account one in the trie - acc := &Account{Balance: big.NewInt(1), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()} + helper := newHelper() + { + // Account one in the trie + stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), + []string{"key-1", "key-2", "key-3"}, + []string{"val-1", "val-2", "val-3"}, + true, + ) + acc := &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()} val, _ := rlp.EncodeToBytes(acc) - accTrie.Update([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e + helper.accTrie.Update([]byte("acc-1"), val) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e + // Identical in the snap key := hashData([]byte("acc-1")) - rawdb.WriteAccountSnapshot(diskdb, key, val) - rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("key-1")), []byte("val-1")) - rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("key-2")), []byte("val-2")) - rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("key-3")), []byte("val-3")) + rawdb.WriteAccountSnapshot(helper.diskdb, key, val) + rawdb.WriteStorageSnapshot(helper.diskdb, key, hashData([]byte("key-1")), []byte("val-1")) + rawdb.WriteStorageSnapshot(helper.diskdb, key, hashData([]byte("key-2")), []byte("val-2")) + rawdb.WriteStorageSnapshot(helper.diskdb, key, hashData([]byte("key-3")), []byte("val-3")) } - { // 100 accounts exist only in snapshot + { + // 100 accounts exist only in snapshot for i := 0; i < 1000; i++ { //acc := &Account{Balance: big.NewInt(int64(i)), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()} acc := &Account{Balance: big.NewInt(int64(i)), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()} val, _ := rlp.EncodeToBytes(acc) key := hashData([]byte(fmt.Sprintf("acc-%d", i))) - rawdb.WriteAccountSnapshot(diskdb, key, val) + rawdb.WriteAccountSnapshot(helper.diskdb, key, val) } } - root, _, _ := accTrie.Commit(nil) - t.Logf("root: %x", root) - triedb.Commit(root, false, nil) - - snap := generateSnapshot(diskdb, triedb, 16, root) + root, snap := helper.CommitAndGenerate() select { case <-snap.genPending: // Snapshot generation succeeded @@ -675,31 +609,22 @@ func TestGenerateWithExtraBeforeAndAfter(t *testing.T) { if false { enableLogging() } - var ( - diskdb = memorydb.New() - triedb = trie.NewDatabase(diskdb) - ) - accTrie, _ := trie.New(common.Hash{}, triedb) + helper := newHelper() { acc := &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()} val, _ := rlp.EncodeToBytes(acc) - accTrie.Update(common.HexToHash("0x03").Bytes(), val) - accTrie.Update(common.HexToHash("0x07").Bytes(), val) - - rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x01"), val) - rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x02"), val) - rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x03"), val) - rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x04"), val) - rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x05"), val) - rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x06"), val) - rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x07"), val) - } - - root, _, _ := accTrie.Commit(nil) - t.Logf("root: %x", root) - triedb.Commit(root, false, nil) - - snap := generateSnapshot(diskdb, triedb, 16, root) + helper.accTrie.Update(common.HexToHash("0x03").Bytes(), val) + helper.accTrie.Update(common.HexToHash("0x07").Bytes(), val) + + rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x01"), val) + rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x02"), val) + rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x03"), val) + rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x04"), val) + rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x05"), val) + rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x06"), val) + rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x07"), val) + } + root, snap := helper.CommitAndGenerate() select { case <-snap.genPending: // Snapshot generation succeeded @@ -721,29 +646,20 @@ func TestGenerateWithMalformedSnapdata(t *testing.T) { if false { enableLogging() } - var ( - diskdb = memorydb.New() - triedb = trie.NewDatabase(diskdb) - ) - accTrie, _ := trie.New(common.Hash{}, triedb) + helper := newHelper() { acc := &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()} val, _ := rlp.EncodeToBytes(acc) - accTrie.Update(common.HexToHash("0x03").Bytes(), val) + helper.accTrie.Update(common.HexToHash("0x03").Bytes(), val) junk := make([]byte, 100) copy(junk, []byte{0xde, 0xad}) - rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x02"), junk) - rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x03"), junk) - rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x04"), junk) - rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x05"), junk) + rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x02"), junk) + rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x03"), junk) + rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x04"), junk) + rawdb.WriteAccountSnapshot(helper.diskdb, common.HexToHash("0x05"), junk) } - - root, _, _ := accTrie.Commit(nil) - t.Logf("root: %x", root) - triedb.Commit(root, false, nil) - - snap := generateSnapshot(diskdb, triedb, 16, root) + root, snap := helper.CommitAndGenerate() select { case <-snap.genPending: // Snapshot generation succeeded @@ -757,7 +673,7 @@ func TestGenerateWithMalformedSnapdata(t *testing.T) { snap.genAbort <- stop <-stop // If we now inspect the snap db, there should exist no extraneous storage items - if data := rawdb.ReadStorageSnapshot(diskdb, hashData([]byte("acc-2")), hashData([]byte("b-key-1"))); data != nil { + if data := rawdb.ReadStorageSnapshot(helper.diskdb, hashData([]byte("acc-2")), hashData([]byte("b-key-1"))); data != nil { t.Fatalf("expected slot to be removed, got %v", string(data)) } } @@ -767,13 +683,13 @@ func TestGenerateFromEmptySnap(t *testing.T) { accountCheckRange = 10 storageCheckRange = 20 helper := newHelper() - stRoot := helper.makeStorageTrie([]string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) // Add 1K accounts to the trie for i := 0; i < 400; i++ { + stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte(fmt.Sprintf("acc-%d", i))), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addTrieAccount(fmt.Sprintf("acc-%d", i), &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) } - root, snap := helper.Generate() + root, snap := helper.CommitAndGenerate() t.Logf("Root: %#x\n", root) // Root: 0x6f7af6d2e1a1bf2b84a3beb3f8b64388465fbc1e274ca5d5d3fc787ca78f59e4 select { @@ -802,12 +718,12 @@ func TestGenerateWithIncompleteStorage(t *testing.T) { helper := newHelper() stKeys := []string{"1", "2", "3", "4", "5", "6", "7", "8"} stVals := []string{"v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8"} - stRoot := helper.makeStorageTrie(stKeys, stVals) // We add 8 accounts, each one is missing exactly one of the storage slots. This means // we don't have to order the keys and figure out exactly which hash-key winds up // on the sensitive spots at the boundaries for i := 0; i < 8; i++ { accKey := fmt.Sprintf("acc-%d", i) + stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte(accKey)), stKeys, stVals, true) helper.addAccount(accKey, &Account{Balance: big.NewInt(int64(i)), Root: stRoot, CodeHash: emptyCode.Bytes()}) var moddedKeys []string var moddedVals []string @@ -819,8 +735,7 @@ func TestGenerateWithIncompleteStorage(t *testing.T) { } helper.addSnapStorage(accKey, moddedKeys, moddedVals) } - - root, snap := helper.Generate() + root, snap := helper.CommitAndGenerate() t.Logf("Root: %#x\n", root) // Root: 0xca73f6f05ba4ca3024ef340ef3dfca8fdabc1b677ff13f5a9571fd49c16e67ff select { @@ -899,10 +814,12 @@ func populateDangling(disk ethdb.KeyValueStore) { // This test will populate some dangling storages to see if they can be cleaned up. func TestGenerateCompleteSnapshotWithDanglingStorage(t *testing.T) { var helper = newHelper() - stRoot := helper.makeStorageTrie([]string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) + stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) helper.addAccount("acc-2", &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) + + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addAccount("acc-3", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) helper.addSnapStorage("acc-1", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) @@ -910,7 +827,7 @@ func TestGenerateCompleteSnapshotWithDanglingStorage(t *testing.T) { populateDangling(helper.diskdb) - root, snap := helper.Generate() + root, snap := helper.CommitAndGenerate() select { case <-snap.genPending: // Snapshot generation succeeded @@ -932,15 +849,17 @@ func TestGenerateCompleteSnapshotWithDanglingStorage(t *testing.T) { // This test will populate some dangling storages to see if they can be cleaned up. func TestGenerateBrokenSnapshotWithDanglingStorage(t *testing.T) { var helper = newHelper() - stRoot := helper.makeStorageTrie([]string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) + stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) + + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: emptyCode.Bytes()}) populateDangling(helper.diskdb) - root, snap := helper.Generate() + root, snap := helper.CommitAndGenerate() select { case <-snap.genPending: // Snapshot generation succeeded diff --git a/core/state/snapshot/iterator_fast.go b/core/state/snapshot/iterator_fast.go index 48069b8fcf5c0..435c28e96f9e8 100644 --- a/core/state/snapshot/iterator_fast.go +++ b/core/state/snapshot/iterator_fast.go @@ -319,7 +319,7 @@ func (fi *fastIterator) Slot() []byte { } // Release iterates over all the remaining live layer iterators and releases each -// of thme individually. +// of them individually. func (fi *fastIterator) Release() { for _, it := range fi.iterators { it.it.Release() @@ -327,7 +327,7 @@ func (fi *fastIterator) Release() { fi.iterators = nil } -// Debug is a convencience helper during testing +// Debug is a convenience helper during testing func (fi *fastIterator) Debug() { for _, it := range fi.iterators { fmt.Printf("[p=%v v=%v] ", it.priority, it.it.Hash()[0]) diff --git a/core/state/snapshot/journal.go b/core/state/snapshot/journal.go index 6836a574090cd..c1a4cc3d47e5a 100644 --- a/core/state/snapshot/journal.go +++ b/core/state/snapshot/journal.go @@ -108,48 +108,19 @@ func loadAndParseJournal(db ethdb.KeyValueStore, base *diskLayer) (snapshot, jou // So if there is no journal, or the journal is invalid(e.g. the journal // is not matched with disk layer; or the it's the legacy-format journal, // etc.), we just discard all diffs and try to recover them later. - journal := rawdb.ReadSnapshotJournal(db) - if len(journal) == 0 { - log.Warn("Loaded snapshot journal", "diskroot", base.root, "diffs", "missing") - return base, generator, nil - } - r := rlp.NewStream(bytes.NewReader(journal), 0) - - // Firstly, resolve the first element as the journal version - version, err := r.Uint() + var current snapshot = base + err := iterateJournal(db, func(parent common.Hash, root common.Hash, destructSet map[common.Hash]struct{}, accountData map[common.Hash][]byte, storageData map[common.Hash]map[common.Hash][]byte) error { + current = newDiffLayer(current, root, destructSet, accountData, storageData) + return nil + }) if err != nil { - log.Warn("Failed to resolve the journal version", "error", err) - return base, generator, nil - } - if version != journalVersion { - log.Warn("Discarded the snapshot journal with wrong version", "required", journalVersion, "got", version) - return base, generator, nil - } - // Secondly, resolve the disk layer root, ensure it's continuous - // with disk layer. Note now we can ensure it's the snapshot journal - // correct version, so we expect everything can be resolved properly. - var root common.Hash - if err := r.Decode(&root); err != nil { - return nil, journalGenerator{}, errors.New("missing disk layer root") - } - // The diff journal is not matched with disk, discard them. - // It can happen that Geth crashes without persisting the latest - // diff journal. - if !bytes.Equal(root.Bytes(), base.root.Bytes()) { - log.Warn("Loaded snapshot journal", "diskroot", base.root, "diffs", "unmatched") return base, generator, nil } - // Load all the snapshot diffs from the journal - snapshot, err := loadDiffLayer(base, r) - if err != nil { - return nil, journalGenerator{}, err - } - log.Debug("Loaded snapshot journal", "diskroot", base.root, "diffhead", snapshot.Root()) - return snapshot, generator, nil + return current, generator, nil } // loadSnapshot loads a pre-existing state snapshot backed by a key-value store. -func loadSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, root common.Hash, recovery bool) (snapshot, bool, error) { +func loadSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, root common.Hash, cache int, recovery bool, noBuild bool) (snapshot, bool, error) { // If snapshotting is disabled (initial sync in progress), don't do anything, // wait for the chain to permit us to do something meaningful if rawdb.ReadSnapshotDisabled(diskdb) { @@ -169,7 +140,7 @@ func loadSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, } snapshot, generator, err := loadAndParseJournal(diskdb, base) if err != nil { - log.Warn("Failed to load new-format journal", "error", err) + log.Warn("Failed to load journal", "error", err) return nil, false, err } // Entire snapshot journal loaded, sanity check the head. If the loaded @@ -193,13 +164,16 @@ func loadSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, // disk layer. log.Warn("Snapshot is not continuous with chain", "snaproot", head, "chainroot", root) } - // Everything loaded correctly, resume any suspended operations + // Load the disk layer status from the generator if it's not complete if !generator.Done { - // Whether or not wiping was in progress, load any generator progress too base.genMarker = generator.Marker if base.genMarker == nil { base.genMarker = []byte{} } + } + // Everything loaded correctly, resume any suspended operations + // if the background generation is allowed + if !generator.Done && !noBuild { base.genPending = make(chan struct{}) base.genAbort = make(chan chan *generatorStats) @@ -218,57 +192,6 @@ func loadSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, return snapshot, false, nil } -// loadDiffLayer reads the next sections of a snapshot journal, reconstructing a new -// diff and verifying that it can be linked to the requested parent. -func loadDiffLayer(parent snapshot, r *rlp.Stream) (snapshot, error) { - // Read the next diff journal entry - var root common.Hash - if err := r.Decode(&root); err != nil { - // The first read may fail with EOF, marking the end of the journal - if err == io.EOF { - return parent, nil - } - return nil, fmt.Errorf("load diff root: %v", err) - } - var destructs []journalDestruct - if err := r.Decode(&destructs); err != nil { - return nil, fmt.Errorf("load diff destructs: %v", err) - } - destructSet := make(map[common.Hash]struct{}) - for _, entry := range destructs { - destructSet[entry.Hash] = struct{}{} - } - var accounts []journalAccount - if err := r.Decode(&accounts); err != nil { - return nil, fmt.Errorf("load diff accounts: %v", err) - } - accountData := make(map[common.Hash][]byte) - for _, entry := range accounts { - if len(entry.Blob) > 0 { // RLP loses nil-ness, but `[]byte{}` is not a valid item, so reinterpret that - accountData[entry.Hash] = entry.Blob - } else { - accountData[entry.Hash] = nil - } - } - var storage []journalStorage - if err := r.Decode(&storage); err != nil { - return nil, fmt.Errorf("load diff storage: %v", err) - } - storageData := make(map[common.Hash]map[common.Hash][]byte) - for _, entry := range storage { - slots := make(map[common.Hash][]byte) - for i, key := range entry.Keys { - if len(entry.Vals[i]) > 0 { // RLP loses nil-ness, but `[]byte{}` is not a valid item, so reinterpret that - slots[key] = entry.Vals[i] - } else { - slots[key] = nil - } - } - storageData[entry.Hash] = slots - } - return loadDiffLayer(newDiffLayer(parent, root, destructSet, accountData, storageData), r) -} - // Journal terminates any in-progress snapshot generation, also implicitly pushing // the progress into the database. func (dl *diskLayer) Journal(buffer *bytes.Buffer) (common.Hash, error) { @@ -345,3 +268,96 @@ func (dl *diffLayer) Journal(buffer *bytes.Buffer) (common.Hash, error) { log.Debug("Journalled diff layer", "root", dl.root, "parent", dl.parent.Root()) return base, nil } + +// journalCallback is a function which is invoked by iterateJournal, every +// time a difflayer is loaded from disk. +type journalCallback = func(parent common.Hash, root common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) error + +// iterateJournal iterates through the journalled difflayers, loading them from +// the database, and invoking the callback for each loaded layer. +// The order is incremental; starting with the bottom-most difflayer, going towards +// the most recent layer. +// This method returns error either if there was some error reading from disk, +// OR if the callback returns an error when invoked. +func iterateJournal(db ethdb.KeyValueReader, callback journalCallback) error { + journal := rawdb.ReadSnapshotJournal(db) + if len(journal) == 0 { + log.Warn("Loaded snapshot journal", "diffs", "missing") + return nil + } + r := rlp.NewStream(bytes.NewReader(journal), 0) + // Firstly, resolve the first element as the journal version + version, err := r.Uint64() + if err != nil { + log.Warn("Failed to resolve the journal version", "error", err) + return errors.New("failed to resolve journal version") + } + if version != journalVersion { + log.Warn("Discarded the snapshot journal with wrong version", "required", journalVersion, "got", version) + return errors.New("wrong journal version") + } + // Secondly, resolve the disk layer root, ensure it's continuous + // with disk layer. Note now we can ensure it's the snapshot journal + // correct version, so we expect everything can be resolved properly. + var parent common.Hash + if err := r.Decode(&parent); err != nil { + return errors.New("missing disk layer root") + } + if baseRoot := rawdb.ReadSnapshotRoot(db); baseRoot != parent { + log.Warn("Loaded snapshot journal", "diskroot", baseRoot, "diffs", "unmatched") + return fmt.Errorf("mismatched disk and diff layers") + } + for { + var ( + root common.Hash + destructs []journalDestruct + accounts []journalAccount + storage []journalStorage + destructSet = make(map[common.Hash]struct{}) + accountData = make(map[common.Hash][]byte) + storageData = make(map[common.Hash]map[common.Hash][]byte) + ) + // Read the next diff journal entry + if err := r.Decode(&root); err != nil { + // The first read may fail with EOF, marking the end of the journal + if errors.Is(err, io.EOF) { + return nil + } + return fmt.Errorf("load diff root: %v", err) + } + if err := r.Decode(&destructs); err != nil { + return fmt.Errorf("load diff destructs: %v", err) + } + if err := r.Decode(&accounts); err != nil { + return fmt.Errorf("load diff accounts: %v", err) + } + if err := r.Decode(&storage); err != nil { + return fmt.Errorf("load diff storage: %v", err) + } + for _, entry := range destructs { + destructSet[entry.Hash] = struct{}{} + } + for _, entry := range accounts { + if len(entry.Blob) > 0 { // RLP loses nil-ness, but `[]byte{}` is not a valid item, so reinterpret that + accountData[entry.Hash] = entry.Blob + } else { + accountData[entry.Hash] = nil + } + } + for _, entry := range storage { + slots := make(map[common.Hash][]byte) + for i, key := range entry.Keys { + if len(entry.Vals[i]) > 0 { // RLP loses nil-ness, but `[]byte{}` is not a valid item, so reinterpret that + slots[key] = entry.Vals[i] + } else { + slots[key] = nil + } + } + storageData[entry.Hash] = slots + } + if err := callback(parent, root, destructSet, accountData, storageData); err != nil { + return err + } + parent = root + } +} diff --git a/core/state/snapshot/snapshot.go b/core/state/snapshot/snapshot.go index 76200851e4696..ee18f4bcdcf6d 100644 --- a/core/state/snapshot/snapshot.go +++ b/core/state/snapshot/snapshot.go @@ -148,6 +148,14 @@ type snapshot interface { StorageIterator(account common.Hash, seek common.Hash) (StorageIterator, bool) } +// Config includes the configurations for snapshots. +type Config struct { + CacheSize int // Megabytes permitted to use for read caches + Recovery bool // Indicator that the snapshots is in the recovery mode + NoBuild bool // Indicator that the snapshots generation is disallowed + AsyncBuild bool // The snapshot generation is allowed to be constructed asynchronously +} + // Tree is an Ethereum state snapshot tree. It consists of one persistent base // layer backed by a key-value store, on top of which arbitrarily many in-memory // diff layers are topped. The memory diffs can form a tree with branching, but @@ -158,9 +166,9 @@ type snapshot interface { // storage data to avoid expensive multi-level trie lookups; and to allow sorted, // cheap iteration of the account/storage tries for sync aid. type Tree struct { + config Config // Snapshots configurations diskdb ethdb.KeyValueStore // Persistent database to store the snapshot triedb *trie.Database // In-memory cache to access the trie through - cache int // Megabytes permitted to use for read caches layers map[common.Hash]snapshot // Collection of all known layers lock sync.RWMutex @@ -179,30 +187,31 @@ type Tree struct { // If the memory layers in the journal do not match the disk layer (e.g. there is // a gap) or the journal is missing, there are two repair cases: // -// - if the 'recovery' parameter is true, all memory diff-layers will be discarded. -// This case happens when the snapshot is 'ahead' of the state trie. -// - otherwise, the entire snapshot is considered invalid and will be recreated on -// a background thread. -func New(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, root common.Hash, async bool, rebuild bool, recovery bool) (*Tree, error) { +// - if the 'recovery' parameter is true, all memory diff-layers will be discarded. +// This case happens when the snapshot is 'ahead' of the state trie. +// - otherwise, the entire snapshot is considered invalid and will be recreated on +// a background thread. +func New(config Config, diskdb ethdb.KeyValueStore, triedb *trie.Database, root common.Hash) (*Tree, error) { // Create a new, empty snapshot tree snap := &Tree{ + config: config, diskdb: diskdb, triedb: triedb, - cache: cache, layers: make(map[common.Hash]snapshot), } - if !async { + // Create the building waiter iff the background generation is allowed + if !config.NoBuild && !config.AsyncBuild { defer snap.waitBuild() } // Attempt to load a previously persisted snapshot and rebuild one if failed - head, disabled, err := loadSnapshot(diskdb, triedb, cache, root, recovery) + head, disabled, err := loadSnapshot(diskdb, triedb, root, config.CacheSize, config.Recovery, config.NoBuild) if disabled { log.Warn("Snapshot maintenance disabled (syncing)") return snap, nil } if err != nil { - if rebuild { - log.Warn("Failed to load snapshot, regenerating", "err", err) + log.Warn("Failed to load snapshot", "err", err) + if !config.NoBuild { snap.Rebuild(root) return snap, nil } @@ -727,7 +736,7 @@ func (t *Tree) Rebuild(root common.Hash) { // generator will run a wiper first if there's not one running right now. log.Info("Rebuilding state snapshot") t.layers = map[common.Hash]snapshot{ - root: generateSnapshot(t.diskdb, t.triedb, t.cache, root), + root: generateSnapshot(t.diskdb, t.triedb, t.config.CacheSize, root), } } diff --git a/core/state/snapshot/snapshot_test.go b/core/state/snapshot/snapshot_test.go index bc4e5cbd04620..7c8077b652ed7 100644 --- a/core/state/snapshot/snapshot_test.go +++ b/core/state/snapshot/snapshot_test.go @@ -265,7 +265,7 @@ func TestPostCapBasicDataAccess(t *testing.T) { snaps.Update(common.HexToHash("0xa3"), common.HexToHash("0xa2"), nil, setAccount("0xa3"), nil) snaps.Update(common.HexToHash("0xb3"), common.HexToHash("0xb2"), nil, setAccount("0xb3"), nil) - // checkExist verifies if an account exiss in a snapshot + // checkExist verifies if an account exists in a snapshot checkExist := func(layer *diffLayer, key string) error { if data, _ := layer.Account(common.HexToHash(key)); data == nil { return fmt.Errorf("expected %x to exist, got nil", common.HexToHash(key)) diff --git a/core/state/snapshot/dangling.go b/core/state/snapshot/utils.go similarity index 55% rename from core/state/snapshot/dangling.go rename to core/state/snapshot/utils.go index ca73da793f7a4..fa1f216e6826f 100644 --- a/core/state/snapshot/dangling.go +++ b/core/state/snapshot/utils.go @@ -18,9 +18,7 @@ package snapshot import ( "bytes" - "errors" "fmt" - "io" "time" "github.com/ethereum/go-ethereum/common" @@ -34,7 +32,7 @@ import ( // storage also has corresponding account data. func CheckDanglingStorage(chaindb ethdb.KeyValueStore) error { if err := checkDanglingDiskStorage(chaindb); err != nil { - return err + log.Error("Database check error", "err", err) } return checkDanglingMemStorage(chaindb) } @@ -75,81 +73,80 @@ func checkDanglingDiskStorage(chaindb ethdb.KeyValueStore) error { // checkDanglingMemStorage checks if there is any 'dangling' storage in the journalled // snapshot difflayers. func checkDanglingMemStorage(db ethdb.KeyValueStore) error { - var ( - start = time.Now() - journal = rawdb.ReadSnapshotJournal(db) - ) - if len(journal) == 0 { - log.Warn("Loaded snapshot journal", "diffs", "missing") + start := time.Now() + log.Info("Checking dangling journalled storage") + err := iterateJournal(db, func(pRoot, root common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) error { + for accHash := range storage { + if _, ok := accounts[accHash]; !ok { + log.Error("Dangling storage - missing account", "account", fmt.Sprintf("%#x", accHash), "root", root) + } + } return nil - } - r := rlp.NewStream(bytes.NewReader(journal), 0) - // Firstly, resolve the first element as the journal version - version, err := r.Uint() + }) if err != nil { - log.Warn("Failed to resolve the journal version", "error", err) - return nil - } - if version != journalVersion { - log.Warn("Discarded the snapshot journal with wrong version", "required", journalVersion, "got", version) - return nil - } - // Secondly, resolve the disk layer root, ensure it's continuous - // with disk layer. Note now we can ensure it's the snapshot journal - // correct version, so we expect everything can be resolved properly. - var root common.Hash - if err := r.Decode(&root); err != nil { - return errors.New("missing disk layer root") - } - // The diff journal is not matched with disk, discard them. - // It can happen that Geth crashes without persisting the latest - // diff journal. - // Load all the snapshot diffs from the journal - if err := checkDanglingJournalStorage(r); err != nil { + log.Info("Failed to resolve snapshot journal", "err", err) return err } log.Info("Verified the snapshot journalled storage", "time", common.PrettyDuration(time.Since(start))) return nil } -// loadDiffLayer reads the next sections of a snapshot journal, reconstructing a new -// diff and verifying that it can be linked to the requested parent. -func checkDanglingJournalStorage(r *rlp.Stream) error { - for { - // Read the next diff journal entry - var root common.Hash - if err := r.Decode(&root); err != nil { - // The first read may fail with EOF, marking the end of the journal - if err == io.EOF { - return nil - } - return fmt.Errorf("load diff root: %v", err) +// CheckJournalAccount shows information about an account, from the disk layer and +// up through the diff layers. +func CheckJournalAccount(db ethdb.KeyValueStore, hash common.Hash) error { + // Look up the disk layer first + baseRoot := rawdb.ReadSnapshotRoot(db) + fmt.Printf("Disklayer: Root: %x\n", baseRoot) + if data := rawdb.ReadAccountSnapshot(db, hash); data != nil { + account := new(Account) + if err := rlp.DecodeBytes(data, account); err != nil { + panic(err) } - var destructs []journalDestruct - if err := r.Decode(&destructs); err != nil { - return fmt.Errorf("load diff destructs: %v", err) + fmt.Printf("\taccount.nonce: %d\n", account.Nonce) + fmt.Printf("\taccount.balance: %x\n", account.Balance) + fmt.Printf("\taccount.root: %x\n", account.Root) + fmt.Printf("\taccount.codehash: %x\n", account.CodeHash) + } + // Check storage + { + it := rawdb.NewKeyLengthIterator(db.NewIterator(append(rawdb.SnapshotStoragePrefix, hash.Bytes()...), nil), 1+2*common.HashLength) + fmt.Printf("\tStorage:\n") + for it.Next() { + slot := it.Key()[33:] + fmt.Printf("\t\t%x: %x\n", slot, it.Value()) } - var accounts []journalAccount - if err := r.Decode(&accounts); err != nil { - return fmt.Errorf("load diff accounts: %v", err) + it.Release() + } + var depth = 0 + + return iterateJournal(db, func(pRoot, root common.Hash, destructs map[common.Hash]struct{}, accounts map[common.Hash][]byte, storage map[common.Hash]map[common.Hash][]byte) error { + _, a := accounts[hash] + _, b := destructs[hash] + _, c := storage[hash] + depth++ + if !a && !b && !c { + return nil } - accountData := make(map[common.Hash][]byte) - for _, entry := range accounts { - if len(entry.Blob) > 0 { // RLP loses nil-ness, but `[]byte{}` is not a valid item, so reinterpret that - accountData[entry.Hash] = entry.Blob - } else { - accountData[entry.Hash] = nil + fmt.Printf("Disklayer+%d: Root: %x, parent %x\n", depth, root, pRoot) + if data, ok := accounts[hash]; ok { + account := new(Account) + if err := rlp.DecodeBytes(data, account); err != nil { + panic(err) } + fmt.Printf("\taccount.nonce: %d\n", account.Nonce) + fmt.Printf("\taccount.balance: %x\n", account.Balance) + fmt.Printf("\taccount.root: %x\n", account.Root) + fmt.Printf("\taccount.codehash: %x\n", account.CodeHash) } - var storage []journalStorage - if err := r.Decode(&storage); err != nil { - return fmt.Errorf("load diff storage: %v", err) + if _, ok := destructs[hash]; ok { + fmt.Printf("\t Destructed!") } - for _, entry := range storage { - if _, ok := accountData[entry.Hash]; !ok { - log.Error("Dangling storage - missing account", "account", fmt.Sprintf("%#x", entry.Hash), "root", root) - return fmt.Errorf("dangling journal snapshot storage account %#x", entry.Hash) + if data, ok := storage[hash]; ok { + fmt.Printf("\tStorage\n") + for k, v := range data { + fmt.Printf("\t\t%x: %x\n", k, v) } } - } + return nil + }) } diff --git a/core/state/state_object.go b/core/state/state_object.go index bcb6dca4f56bc..178b930593179 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -28,6 +28,7 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie" ) var emptyCodeHash = crypto.Keccak256(nil) @@ -49,7 +50,7 @@ func (s Storage) String() (str string) { } func (s Storage) Copy() Storage { - cpy := make(Storage) + cpy := make(Storage, len(s)) for key, value := range s { cpy[key] = value } @@ -154,13 +155,13 @@ func (s *stateObject) getTrie(db Database) Trie { if s.data.Root != emptyRoot && s.db.prefetcher != nil { // When the miner is creating the pending state, there is no // prefetcher - s.trie = s.db.prefetcher.trie(s.data.Root) + s.trie = s.db.prefetcher.trie(s.addrHash, s.data.Root) } if s.trie == nil { var err error - s.trie, err = db.OpenStorageTrie(s.addrHash, s.data.Root) + s.trie, err = db.OpenStorageTrie(s.db.originalRoot, s.addrHash, s.data.Root) if err != nil { - s.trie, _ = db.OpenStorageTrie(s.addrHash, common.Hash{}) + s.trie, _ = db.OpenStorageTrie(s.db.originalRoot, s.addrHash, common.Hash{}) s.setError(fmt.Errorf("can't create storage trie: %v", err)) } } @@ -295,7 +296,7 @@ func (s *stateObject) finalise(prefetch bool) { } } if s.db.prefetcher != nil && prefetch && len(slotsToPrefetch) > 0 && s.data.Root != emptyRoot { - s.db.prefetcher.prefetch(s.data.Root, slotsToPrefetch) + s.db.prefetcher.prefetch(s.addrHash, s.data.Root, slotsToPrefetch) } if len(s.dirtyStorage) > 0 { s.dirtyStorage = make(Storage) @@ -352,7 +353,7 @@ func (s *stateObject) updateTrie(db Database) Trie { usedStorage = append(usedStorage, common.CopyBytes(key[:])) // Copy needed for closure } if s.db.prefetcher != nil { - s.db.prefetcher.used(s.data.Root, usedStorage) + s.db.prefetcher.used(s.addrHash, s.data.Root, usedStorage) } if len(s.pendingStorage) > 0 { s.pendingStorage = make(Storage) @@ -375,23 +376,23 @@ func (s *stateObject) updateRoot(db Database) { // CommitTrie the storage trie of the object to db. // This updates the trie root. -func (s *stateObject) CommitTrie(db Database) (int, error) { +func (s *stateObject) CommitTrie(db Database) (*trie.NodeSet, error) { // If nothing changed, don't bother with hashing anything if s.updateTrie(db) == nil { - return 0, nil + return nil, nil } if s.dbErr != nil { - return 0, s.dbErr + return nil, s.dbErr } // Track the amount of time wasted on committing the storage trie if metrics.EnabledExpensive { defer func(start time.Time) { s.db.StorageCommits += time.Since(start) }(time.Now()) } - root, committed, err := s.trie.Commit(nil) + root, nodes, err := s.trie.Commit(false) if err == nil { s.data.Root = root } - return committed, err + return nodes, err } // AddBalance adds amount to s's balance. diff --git a/core/state/state_test.go b/core/state/state_test.go index 0a55d7781fd18..b6b46e446fba9 100644 --- a/core/state/state_test.go +++ b/core/state/state_test.go @@ -25,6 +25,7 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/trie" ) type stateTest struct { @@ -40,7 +41,7 @@ func newStateTest() *stateTest { func TestDump(t *testing.T) { db := rawdb.NewMemoryDatabase() - sdb, _ := New(common.Hash{}, NewDatabaseWithConfig(db, nil), nil) + sdb, _ := New(common.Hash{}, NewDatabaseWithConfig(db, &trie.Config{Preimages: true}), nil) s := &stateTest{db: db, state: sdb} // generate a few entries diff --git a/core/state/statedb.go b/core/state/statedb.go index 1d31cf470be03..29a1ccf2d737c 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -62,11 +62,14 @@ func (n *proofList) Delete(key []byte) error { // * Contracts // * Accounts type StateDB struct { - db Database - prefetcher *triePrefetcher - originalRoot common.Hash // The pre-state root, before any changes were made - trie Trie - hasher crypto.KeccakState + db Database + prefetcher *triePrefetcher + trie Trie + hasher crypto.KeccakState + + // originalRoot is the pre-state root, before any changes were made. + // It will be updated when the Commit is called. + originalRoot common.Hash snaps *snapshot.Tree snap snapshot.Snapshot @@ -117,6 +120,7 @@ type StateDB struct { SnapshotAccountReads time.Duration SnapshotStorageReads time.Duration SnapshotCommits time.Duration + TrieDBCommits time.Duration AccountUpdated int StorageUpdated int @@ -481,7 +485,7 @@ func (s *StateDB) deleteStateObject(obj *stateObject) { } // Delete the account from the trie addr := obj.Address() - if err := s.trie.TryDelete(addr[:]); err != nil { + if err := s.trie.TryDeleteAccount(addr[:]); err != nil { s.setError(fmt.Errorf("deleteStateObject (%x) error: %v", addr[:], err)) } } @@ -534,20 +538,16 @@ func (s *StateDB) getDeletedStateObject(addr common.Address) *stateObject { // If snapshot unavailable or reading from it failed, load from the database if data == nil { start := time.Now() - enc, err := s.trie.TryGet(addr.Bytes()) + var err error + data, err = s.trie.TryGetAccount(addr.Bytes()) if metrics.EnabledExpensive { s.AccountReads += time.Since(start) } if err != nil { - s.setError(fmt.Errorf("getDeleteStateObject (%x) error: %v", addr.Bytes(), err)) - return nil - } - if len(enc) == 0 { + s.setError(fmt.Errorf("getDeleteStateObject (%x) error: %w", addr.Bytes(), err)) return nil } - data = new(types.StateAccount) - if err := rlp.DecodeBytes(enc, data); err != nil { - log.Error("Failed to decode state object", "addr", addr, "err", err) + if data == nil { return nil } } @@ -601,8 +601,8 @@ func (s *StateDB) createObject(addr common.Address) (newobj, prev *stateObject) // CreateAccount is called during the EVM CREATE operation. The situation might arise that // a contract does the following: // -// 1. sends funds to sha(account ++ (nonce + 1)) -// 2. tx_create(sha(account ++ nonce)) (note that this gets the address of 1) +// 1. sends funds to sha(account ++ (nonce + 1)) +// 2. tx_create(sha(account ++ nonce)) (note that this gets the address of 1) // // Carrying over the balance ensures that Ether doesn't disappear. func (s *StateDB) CreateAccount(addr common.Address) { @@ -648,6 +648,7 @@ func (s *StateDB) Copy() *StateDB { state := &StateDB{ db: s.db, trie: s.db.CopyTrie(s.trie), + originalRoot: s.originalRoot, stateObjects: make(map[common.Address]*stateObject, len(s.journal.dirties)), stateObjectsPending: make(map[common.Address]struct{}, len(s.stateObjectsPending)), stateObjectsDirty: make(map[common.Address]struct{}, len(s.journal.dirties)), @@ -770,7 +771,7 @@ func (s *StateDB) GetRefund() uint64 { return s.refund } -// Finalise finalises the state by removing the s destructed objects and clears +// Finalise finalises the state by removing the destructed objects and clears // the journal as well as the refunds. Finalise, however, will not push any updates // into the tries just yet. Only IntermediateRoot or Commit will do that. func (s *StateDB) Finalise(deleteEmptyObjects bool) { @@ -792,7 +793,7 @@ func (s *StateDB) Finalise(deleteEmptyObjects bool) { // If state snapshotting is active, also mark the destruction there. // Note, we can't do this only at the end of a block because multiple // transactions within the same block might self destruct and then - // ressurrect an account; but the snapshotter needs both events. + // resurrect an account; but the snapshotter needs both events. if s.snap != nil { s.snapDestructs[obj.addrHash] = struct{}{} // We need to maintain account deletions explicitly (will remain set indefinitely) delete(s.snapAccounts, obj.addrHash) // Clear out any previously updated account data (may be recreated via a ressurrect) @@ -810,7 +811,7 @@ func (s *StateDB) Finalise(deleteEmptyObjects bool) { addressesToPrefetch = append(addressesToPrefetch, common.CopyBytes(addr[:])) // Copy needed for closure } if s.prefetcher != nil && len(addressesToPrefetch) > 0 { - s.prefetcher.prefetch(s.originalRoot, addressesToPrefetch) + s.prefetcher.prefetch(common.Hash{}, s.originalRoot, addressesToPrefetch) } // Invalidate journal because reverting across transactions is not allowed. s.clearJournalAndRefund() @@ -840,7 +841,7 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash { // Although naively it makes sense to retrieve the account trie and then do // the contract storage and account updates sequentially, that short circuits // the account prefetcher. Instead, let's process all the storage updates - // first, giving the account prefeches just a few more milliseconds of time + // first, giving the account prefetches just a few more milliseconds of time // to pull useful data from disk. for addr := range s.stateObjectsPending { if obj := s.stateObjects[addr]; !obj.deleted { @@ -851,7 +852,7 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash { // _untouched_. We can check with the prefetcher, if it can give us a trie // which has the same root, but also has some content loaded into it. if prefetcher != nil { - if trie := prefetcher.trie(s.originalRoot); trie != nil { + if trie := prefetcher.trie(common.Hash{}, s.originalRoot); trie != nil { s.trie = trie } } @@ -867,7 +868,7 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash { usedAddrs = append(usedAddrs, common.CopyBytes(addr[:])) // Copy needed for closure } if prefetcher != nil { - prefetcher.used(s.originalRoot, usedAddrs) + prefetcher.used(common.Hash{}, s.originalRoot, usedAddrs) } if len(s.stateObjectsPending) > 0 { s.stateObjectsPending = make(map[common.Address]struct{}) @@ -891,7 +892,7 @@ func (s *StateDB) clearJournalAndRefund() { s.journal = newJournal() s.refund = 0 } - s.validRevisions = s.validRevisions[:0] // Snapshots can be created without journal entires + s.validRevisions = s.validRevisions[:0] // Snapshots can be created without journal entries } // Commit writes the state to the underlying in-memory trie database. @@ -903,8 +904,14 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { s.IntermediateRoot(deleteEmptyObjects) // Commit objects to the trie, measuring the elapsed time - var storageCommitted int - codeWriter := s.db.TrieDB().DiskDB().NewBatch() + var ( + accountTrieNodesUpdated int + accountTrieNodesDeleted int + storageTrieNodesUpdated int + storageTrieNodesDeleted int + nodes = trie.NewMergedNodeSet() + ) + codeWriter := s.db.DiskDB().NewBatch() for addr := range s.stateObjectsDirty { if obj := s.stateObjects[addr]; !obj.deleted { // Write any contract code associated with the state object @@ -913,11 +920,19 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { obj.dirtyCode = false } // Write any storage changes in the state object to its storage trie - committed, err := obj.CommitTrie(s.db) + set, err := obj.CommitTrie(s.db) if err != nil { return common.Hash{}, err } - storageCommitted += committed + // Merge the dirty nodes of storage trie into global set + if set != nil { + if err := nodes.Merge(set); err != nil { + return common.Hash{}, err + } + updates, deleted := set.Size() + storageTrieNodesUpdated += updates + storageTrieNodesDeleted += deleted + } } } if len(s.stateObjectsDirty) > 0 { @@ -928,26 +943,22 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { log.Crit("Failed to commit dirty codes", "error", err) } } - // Write the account trie changes, measuing the amount of wasted time + // Write the account trie changes, measuring the amount of wasted time var start time.Time if metrics.EnabledExpensive { start = time.Now() } - // The onleaf func is called _serially_, so we can reuse the same account - // for unmarshalling every time. - var account types.StateAccount - root, accountCommitted, err := s.trie.Commit(func(_ [][]byte, _ []byte, leaf []byte, parent common.Hash) error { - if err := rlp.DecodeBytes(leaf, &account); err != nil { - return nil - } - if account.Root != emptyRoot { - s.db.TrieDB().Reference(account.Root, parent) - } - return nil - }) + root, set, err := s.trie.Commit(true) if err != nil { return common.Hash{}, err } + // Merge the dirty nodes of account trie into global set + if set != nil { + if err := nodes.Merge(set); err != nil { + return common.Hash{}, err + } + accountTrieNodesUpdated, accountTrieNodesDeleted = set.Size() + } if metrics.EnabledExpensive { s.AccountCommits += time.Since(start) @@ -955,16 +966,16 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { storageUpdatedMeter.Mark(int64(s.StorageUpdated)) accountDeletedMeter.Mark(int64(s.AccountDeleted)) storageDeletedMeter.Mark(int64(s.StorageDeleted)) - accountCommittedMeter.Mark(int64(accountCommitted)) - storageCommittedMeter.Mark(int64(storageCommitted)) + accountTrieUpdatedMeter.Mark(int64(accountTrieNodesUpdated)) + accountTrieDeletedMeter.Mark(int64(accountTrieNodesDeleted)) + storageTriesUpdatedMeter.Mark(int64(storageTrieNodesUpdated)) + storageTriesDeletedMeter.Mark(int64(storageTrieNodesDeleted)) s.AccountUpdated, s.AccountDeleted = 0, 0 s.StorageUpdated, s.StorageDeleted = 0, 0 } // If snapshotting is enabled, update the snapshot tree with this new version if s.snap != nil { - if metrics.EnabledExpensive { - defer func(start time.Time) { s.SnapshotCommits += time.Since(start) }(time.Now()) - } + start := time.Now() // Only update if there's a state transition (skip empty Clique blocks) if parent := s.snap.Root(); parent != root { if err := s.snaps.Update(root, parent, s.snapDestructs, s.snapAccounts, s.snapStorage); err != nil { @@ -978,9 +989,29 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { log.Warn("Failed to cap snapshot tree", "root", root, "layers", 128, "err", err) } } + if metrics.EnabledExpensive { + s.SnapshotCommits += time.Since(start) + } s.snap, s.snapDestructs, s.snapAccounts, s.snapStorage = nil, nil, nil, nil } - return root, err + if root == (common.Hash{}) { + root = emptyRoot + } + origin := s.originalRoot + if origin == (common.Hash{}) { + origin = emptyRoot + } + if root != origin { + start := time.Now() + if err := s.db.TrieDB().Update(nodes); err != nil { + return common.Hash{}, err + } + s.originalRoot = root + if metrics.EnabledExpensive { + s.TrieDBCommits += time.Since(start) + } + } + return root, nil } // PrepareAccessList handles the preparatory steps for executing a state transition with diff --git a/core/state/statedb_test.go b/core/state/statedb_test.go index e9576d4dc44d3..6fe36a7ecffdf 100644 --- a/core/state/statedb_test.go +++ b/core/state/statedb_test.go @@ -699,7 +699,6 @@ func TestDeleteCreateRevert(t *testing.T) { // the Commit operation fails with an error // If we are missing trie nodes, we should not continue writing to the trie func TestMissingTrieNodes(t *testing.T) { - // Create an initial state with a few accounts memDb := rawdb.NewMemoryDatabase() db := NewDatabase(memDb) @@ -772,7 +771,7 @@ func TestStateDBAccessList(t *testing.T) { t.Fatalf("expected %x to be in access list", address) } } - // Check that only the expected addresses are present in the acesslist + // Check that only the expected addresses are present in the access list for address := range state.accessList.addresses { if _, exist := addressMap[address]; !exist { t.Fatalf("extra address %x in access list", address) @@ -915,3 +914,43 @@ func TestStateDBAccessList(t *testing.T) { t.Fatalf("expected empty, got %d", got) } } + +// Tests that account and storage tries are flushed in the correct order and that +// no data loss occurs. +func TestFlushOrderDataLoss(t *testing.T) { + // Create a state trie with many accounts and slots + var ( + memdb = rawdb.NewMemoryDatabase() + statedb = NewDatabase(memdb) + state, _ = New(common.Hash{}, statedb, nil) + ) + for a := byte(0); a < 10; a++ { + state.CreateAccount(common.Address{a}) + for s := byte(0); s < 10; s++ { + state.SetState(common.Address{a}, common.Hash{a, s}, common.Hash{a, s}) + } + } + root, err := state.Commit(false) + if err != nil { + t.Fatalf("failed to commit state trie: %v", err) + } + statedb.TrieDB().Reference(root, common.Hash{}) + if err := statedb.TrieDB().Cap(1024); err != nil { + t.Fatalf("failed to cap trie dirty cache: %v", err) + } + if err := statedb.TrieDB().Commit(root, false, nil); err != nil { + t.Fatalf("failed to commit state trie: %v", err) + } + // Reopen the state trie from flushed disk and verify it + state, err = New(root, NewDatabase(memdb), nil) + if err != nil { + t.Fatalf("failed to reopen state trie: %v", err) + } + for a := byte(0); a < 10; a++ { + for s := byte(0); s < 10; s++ { + if have := state.GetState(common.Address{a}, common.Hash{a, s}); have != (common.Hash{a, s}) { + t.Errorf("account %d: slot %d: state mismatch: have %x, want %x", a, s, have, common.Hash{a, s}) + } + } + } +} diff --git a/core/state/sync.go b/core/state/sync.go index cc7d01a2188dd..00a4c67aa3cb7 100644 --- a/core/state/sync.go +++ b/core/state/sync.go @@ -27,20 +27,20 @@ import ( ) // NewStateSync create a new state trie download scheduler. -func NewStateSync(root common.Hash, database ethdb.KeyValueReader, onLeaf func(paths [][]byte, leaf []byte) error) *trie.Sync { +func NewStateSync(root common.Hash, database ethdb.KeyValueReader, onLeaf func(keys [][]byte, leaf []byte) error) *trie.Sync { // Register the storage slot callback if the external callback is specified. - var onSlot func(paths [][]byte, hexpath []byte, leaf []byte, parent common.Hash) error + var onSlot func(keys [][]byte, path []byte, leaf []byte, parent common.Hash, parentPath []byte) error if onLeaf != nil { - onSlot = func(paths [][]byte, hexpath []byte, leaf []byte, parent common.Hash) error { - return onLeaf(paths, leaf) + onSlot = func(keys [][]byte, path []byte, leaf []byte, parent common.Hash, parentPath []byte) error { + return onLeaf(keys, leaf) } } // Register the account callback to connect the state trie and the storage // trie belongs to the contract. var syncer *trie.Sync - onAccount := func(paths [][]byte, hexpath []byte, leaf []byte, parent common.Hash) error { + onAccount := func(keys [][]byte, path []byte, leaf []byte, parent common.Hash, parentPath []byte) error { if onLeaf != nil { - if err := onLeaf(paths, leaf); err != nil { + if err := onLeaf(keys, leaf); err != nil { return err } } @@ -48,8 +48,8 @@ func NewStateSync(root common.Hash, database ethdb.KeyValueReader, onLeaf func(p if err := rlp.Decode(bytes.NewReader(leaf), &obj); err != nil { return err } - syncer.AddSubTrie(obj.Root, hexpath, parent, onSlot) - syncer.AddCodeEntry(common.BytesToHash(obj.CodeHash), hexpath, parent) + syncer.AddSubTrie(obj.Root, path, parent, parentPath, onSlot) + syncer.AddCodeEntry(common.BytesToHash(obj.CodeHash), path, parent, parentPath) return nil } syncer = trie.NewSync(root, database, onAccount) diff --git a/core/state/sync_test.go b/core/state/sync_test.go index 007590c76d9c7..dbcbb7c963443 100644 --- a/core/state/sync_test.go +++ b/core/state/sync_test.go @@ -100,11 +100,11 @@ func checkStateAccounts(t *testing.T, db ethdb.Database, root common.Hash, accou } // checkTrieConsistency checks that all nodes in a (sub-)trie are indeed present. -func checkTrieConsistency(db ethdb.Database, root common.Hash) error { +func checkTrieConsistency(db ethdb.KeyValueStore, root common.Hash) error { if v, _ := db.Get(root[:]); v == nil { return nil // Consider a non existent state consistent. } - trie, err := trie.New(root, trie.NewDatabase(db)) + trie, err := trie.New(trie.StateTrieID(root), trie.NewDatabase(db)) if err != nil { return err } @@ -134,8 +134,8 @@ func checkStateConsistency(db ethdb.Database, root common.Hash) error { func TestEmptyStateSync(t *testing.T) { empty := common.HexToHash("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421") sync := NewStateSync(empty, rawdb.NewMemoryDatabase(), nil) - if nodes, paths, codes := sync.Missing(1); len(nodes) != 0 || len(paths) != 0 || len(codes) != 0 { - t.Errorf(" content requested for empty state: %v, %v, %v", nodes, paths, codes) + if paths, nodes, codes := sync.Missing(1); len(paths) != 0 || len(nodes) != 0 || len(codes) != 0 { + t.Errorf("content requested for empty state: %v, %v, %v", nodes, paths, codes) } } @@ -160,66 +160,94 @@ func TestIterativeStateSyncBatchedByPath(t *testing.T) { testIterativeStateSync(t, 100, false, true) } +// stateElement represents the element in the state trie(bytecode or trie node). +type stateElement struct { + path string + hash common.Hash + code common.Hash + syncPath trie.SyncPath +} + func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool) { // Create a random state to copy srcDb, srcRoot, srcAccounts := makeTestState() if commit { srcDb.TrieDB().Commit(srcRoot, false, nil) } - srcTrie, _ := trie.New(srcRoot, srcDb.TrieDB()) + srcTrie, _ := trie.New(trie.StateTrieID(srcRoot), srcDb.TrieDB()) // Create a destination state and sync with the scheduler dstDb := rawdb.NewMemoryDatabase() sched := NewStateSync(srcRoot, dstDb, nil) - nodes, paths, codes := sched.Missing(count) var ( - hashQueue []common.Hash - pathQueue []trie.SyncPath + nodeElements []stateElement + codeElements []stateElement ) - if !bypath { - hashQueue = append(append(hashQueue[:0], nodes...), codes...) - } else { - hashQueue = append(hashQueue[:0], codes...) - pathQueue = append(pathQueue[:0], paths...) + paths, nodes, codes := sched.Missing(count) + for i := 0; i < len(paths); i++ { + nodeElements = append(nodeElements, stateElement{ + path: paths[i], + hash: nodes[i], + syncPath: trie.NewSyncPath([]byte(paths[i])), + }) } - for len(hashQueue)+len(pathQueue) > 0 { - results := make([]trie.SyncResult, len(hashQueue)+len(pathQueue)) - for i, hash := range hashQueue { - data, err := srcDb.TrieDB().Node(hash) - if err != nil { - data, err = srcDb.ContractCode(common.Hash{}, hash) - } + for i := 0; i < len(codes); i++ { + codeElements = append(codeElements, stateElement{ + code: codes[i], + }) + } + for len(nodeElements)+len(codeElements) > 0 { + var ( + nodeResults = make([]trie.NodeSyncResult, len(nodeElements)) + codeResults = make([]trie.CodeSyncResult, len(codeElements)) + ) + for i, element := range codeElements { + data, err := srcDb.ContractCode(common.Hash{}, element.code) if err != nil { - t.Fatalf("failed to retrieve node data for hash %x", hash) + t.Fatalf("failed to retrieve contract bytecode for hash %x", element.code) } - results[i] = trie.SyncResult{Hash: hash, Data: data} + codeResults[i] = trie.CodeSyncResult{Hash: element.code, Data: data} } - for i, path := range pathQueue { - if len(path) == 1 { - data, _, err := srcTrie.TryGetNode(path[0]) - if err != nil { - t.Fatalf("failed to retrieve node data for path %x: %v", path, err) + for i, node := range nodeElements { + if bypath { + if len(node.syncPath) == 1 { + data, _, err := srcTrie.TryGetNode(node.syncPath[0]) + if err != nil { + t.Fatalf("failed to retrieve node data for path %x: %v", node.syncPath[0], err) + } + nodeResults[i] = trie.NodeSyncResult{Path: node.path, Data: data} + } else { + var acc types.StateAccount + if err := rlp.DecodeBytes(srcTrie.Get(node.syncPath[0]), &acc); err != nil { + t.Fatalf("failed to decode account on path %x: %v", node.syncPath[0], err) + } + id := trie.StorageTrieID(srcRoot, common.BytesToHash(node.syncPath[0]), acc.Root) + stTrie, err := trie.New(id, srcDb.TrieDB()) + if err != nil { + t.Fatalf("failed to retriev storage trie for path %x: %v", node.syncPath[1], err) + } + data, _, err := stTrie.TryGetNode(node.syncPath[1]) + if err != nil { + t.Fatalf("failed to retrieve node data for path %x: %v", node.syncPath[1], err) + } + nodeResults[i] = trie.NodeSyncResult{Path: node.path, Data: data} } - results[len(hashQueue)+i] = trie.SyncResult{Hash: crypto.Keccak256Hash(data), Data: data} } else { - var acc types.StateAccount - if err := rlp.DecodeBytes(srcTrie.Get(path[0]), &acc); err != nil { - t.Fatalf("failed to decode account on path %x: %v", path, err) - } - stTrie, err := trie.New(acc.Root, srcDb.TrieDB()) + data, err := srcDb.TrieDB().Node(node.hash) if err != nil { - t.Fatalf("failed to retriev storage trie for path %x: %v", path, err) + t.Fatalf("failed to retrieve node data for key %v", []byte(node.path)) } - data, _, err := stTrie.TryGetNode(path[1]) - if err != nil { - t.Fatalf("failed to retrieve node data for path %x: %v", path, err) - } - results[len(hashQueue)+i] = trie.SyncResult{Hash: crypto.Keccak256Hash(data), Data: data} + nodeResults[i] = trie.NodeSyncResult{Path: node.path, Data: data} } } - for _, result := range results { - if err := sched.Process(result); err != nil { + for _, result := range codeResults { + if err := sched.ProcessCode(result); err != nil { + t.Errorf("failed to process result %v", err) + } + } + for _, result := range nodeResults { + if err := sched.ProcessNode(result); err != nil { t.Errorf("failed to process result %v", err) } } @@ -229,12 +257,20 @@ func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool) { } batch.Write() - nodes, paths, codes = sched.Missing(count) - if !bypath { - hashQueue = append(append(hashQueue[:0], nodes...), codes...) - } else { - hashQueue = append(hashQueue[:0], codes...) - pathQueue = append(pathQueue[:0], paths...) + paths, nodes, codes = sched.Missing(count) + nodeElements = nodeElements[:0] + for i := 0; i < len(paths); i++ { + nodeElements = append(nodeElements, stateElement{ + path: paths[i], + hash: nodes[i], + syncPath: trie.NewSyncPath([]byte(paths[i])), + }) + } + codeElements = codeElements[:0] + for i := 0; i < len(codes); i++ { + codeElements = append(codeElements, stateElement{ + code: codes[i], + }) } } // Cross check that the two states are in sync @@ -251,26 +287,58 @@ func TestIterativeDelayedStateSync(t *testing.T) { dstDb := rawdb.NewMemoryDatabase() sched := NewStateSync(srcRoot, dstDb, nil) - nodes, _, codes := sched.Missing(0) - queue := append(append([]common.Hash{}, nodes...), codes...) - - for len(queue) > 0 { + var ( + nodeElements []stateElement + codeElements []stateElement + ) + paths, nodes, codes := sched.Missing(0) + for i := 0; i < len(paths); i++ { + nodeElements = append(nodeElements, stateElement{ + path: paths[i], + hash: nodes[i], + syncPath: trie.NewSyncPath([]byte(paths[i])), + }) + } + for i := 0; i < len(codes); i++ { + codeElements = append(codeElements, stateElement{ + code: codes[i], + }) + } + for len(nodeElements)+len(codeElements) > 0 { // Sync only half of the scheduled nodes - results := make([]trie.SyncResult, len(queue)/2+1) - for i, hash := range queue[:len(results)] { - data, err := srcDb.TrieDB().Node(hash) - if err != nil { - data, err = srcDb.ContractCode(common.Hash{}, hash) + var nodeProcessed int + var codeProcessed int + if len(codeElements) > 0 { + codeResults := make([]trie.CodeSyncResult, len(codeElements)/2+1) + for i, element := range codeElements[:len(codeResults)] { + data, err := srcDb.ContractCode(common.Hash{}, element.code) + if err != nil { + t.Fatalf("failed to retrieve contract bytecode for %x", element.code) + } + codeResults[i] = trie.CodeSyncResult{Hash: element.code, Data: data} } - if err != nil { - t.Fatalf("failed to retrieve node data for %x", hash) + for _, result := range codeResults { + if err := sched.ProcessCode(result); err != nil { + t.Fatalf("failed to process result %v", err) + } } - results[i] = trie.SyncResult{Hash: hash, Data: data} + codeProcessed = len(codeResults) } - for _, result := range results { - if err := sched.Process(result); err != nil { - t.Fatalf("failed to process result %v", err) + if len(nodeElements) > 0 { + nodeResults := make([]trie.NodeSyncResult, len(nodeElements)/2+1) + for i, element := range nodeElements[:len(nodeResults)] { + data, err := srcDb.TrieDB().Node(element.hash) + if err != nil { + t.Fatalf("failed to retrieve contract bytecode for %x", element.code) + } + nodeResults[i] = trie.NodeSyncResult{Path: element.path, Data: data} + } + for _, result := range nodeResults { + if err := sched.ProcessNode(result); err != nil { + t.Fatalf("failed to process result %v", err) + } } + nodeProcessed = len(nodeResults) } batch := dstDb.NewBatch() if err := sched.Commit(batch); err != nil { @@ -278,8 +346,21 @@ func TestIterativeDelayedStateSync(t *testing.T) { } batch.Write() - nodes, _, codes = sched.Missing(0) - queue = append(append(queue[len(results):], nodes...), codes...) + paths, nodes, codes = sched.Missing(0) + nodeElements = nodeElements[nodeProcessed:] + for i := 0; i < len(paths); i++ { + nodeElements = append(nodeElements, stateElement{ + path: paths[i], + hash: nodes[i], + syncPath: trie.NewSyncPath([]byte(paths[i])), + }) + } + codeElements = codeElements[codeProcessed:] + for i := 0; i < len(codes); i++ { + codeElements = append(codeElements, stateElement{ + code: codes[i], + }) + } } // Cross check that the two states are in sync checkStateAccounts(t, dstDb, srcRoot, srcAccounts) @@ -299,40 +380,70 @@ func testIterativeRandomStateSync(t *testing.T, count int) { dstDb := rawdb.NewMemoryDatabase() sched := NewStateSync(srcRoot, dstDb, nil) - queue := make(map[common.Hash]struct{}) - nodes, _, codes := sched.Missing(count) - for _, hash := range append(nodes, codes...) { - queue[hash] = struct{}{} + nodeQueue := make(map[string]stateElement) + codeQueue := make(map[common.Hash]struct{}) + paths, nodes, codes := sched.Missing(count) + for i, path := range paths { + nodeQueue[path] = stateElement{ + path: path, + hash: nodes[i], + syncPath: trie.NewSyncPath([]byte(path)), + } } - for len(queue) > 0 { + for _, hash := range codes { + codeQueue[hash] = struct{}{} + } + for len(nodeQueue)+len(codeQueue) > 0 { // Fetch all the queued nodes in a random order - results := make([]trie.SyncResult, 0, len(queue)) - for hash := range queue { - data, err := srcDb.TrieDB().Node(hash) - if err != nil { - data, err = srcDb.ContractCode(common.Hash{}, hash) + if len(codeQueue) > 0 { + results := make([]trie.CodeSyncResult, 0, len(codeQueue)) + for hash := range codeQueue { + data, err := srcDb.ContractCode(common.Hash{}, hash) + if err != nil { + t.Fatalf("failed to retrieve node data for %x", hash) + } + results = append(results, trie.CodeSyncResult{Hash: hash, Data: data}) } - if err != nil { - t.Fatalf("failed to retrieve node data for %x", hash) + for _, result := range results { + if err := sched.ProcessCode(result); err != nil { + t.Fatalf("failed to process result %v", err) + } } - results = append(results, trie.SyncResult{Hash: hash, Data: data}) } - // Feed the retrieved results back and queue new tasks - for _, result := range results { - if err := sched.Process(result); err != nil { - t.Fatalf("failed to process result %v", err) + if len(nodeQueue) > 0 { + results := make([]trie.NodeSyncResult, 0, len(nodeQueue)) + for path, element := range nodeQueue { + data, err := srcDb.TrieDB().Node(element.hash) + if err != nil { + t.Fatalf("failed to retrieve node data for %x %v %v", element.hash, []byte(element.path), element.path) + } + results = append(results, trie.NodeSyncResult{Path: path, Data: data}) + } + for _, result := range results { + if err := sched.ProcessNode(result); err != nil { + t.Fatalf("failed to process result %v", err) + } } } + // Feed the retrieved results back and queue new tasks batch := dstDb.NewBatch() if err := sched.Commit(batch); err != nil { t.Fatalf("failed to commit data: %v", err) } batch.Write() - queue = make(map[common.Hash]struct{}) - nodes, _, codes = sched.Missing(count) - for _, hash := range append(nodes, codes...) { - queue[hash] = struct{}{} + nodeQueue = make(map[string]stateElement) + codeQueue = make(map[common.Hash]struct{}) + paths, nodes, codes := sched.Missing(count) + for i, path := range paths { + nodeQueue[path] = stateElement{ + path: path, + hash: nodes[i], + syncPath: trie.NewSyncPath([]byte(path)), + } + } + for _, hash := range codes { + codeQueue[hash] = struct{}{} } } // Cross check that the two states are in sync @@ -349,34 +460,62 @@ func TestIterativeRandomDelayedStateSync(t *testing.T) { dstDb := rawdb.NewMemoryDatabase() sched := NewStateSync(srcRoot, dstDb, nil) - queue := make(map[common.Hash]struct{}) - nodes, _, codes := sched.Missing(0) - for _, hash := range append(nodes, codes...) { - queue[hash] = struct{}{} + nodeQueue := make(map[string]stateElement) + codeQueue := make(map[common.Hash]struct{}) + paths, nodes, codes := sched.Missing(0) + for i, path := range paths { + nodeQueue[path] = stateElement{ + path: path, + hash: nodes[i], + syncPath: trie.NewSyncPath([]byte(path)), + } + } + for _, hash := range codes { + codeQueue[hash] = struct{}{} } - for len(queue) > 0 { + for len(nodeQueue)+len(codeQueue) > 0 { // Sync only half of the scheduled nodes, even those in random order - results := make([]trie.SyncResult, 0, len(queue)/2+1) - for hash := range queue { - delete(queue, hash) + if len(codeQueue) > 0 { + results := make([]trie.CodeSyncResult, 0, len(codeQueue)/2+1) + for hash := range codeQueue { + delete(codeQueue, hash) - data, err := srcDb.TrieDB().Node(hash) - if err != nil { - data, err = srcDb.ContractCode(common.Hash{}, hash) + data, err := srcDb.ContractCode(common.Hash{}, hash) + if err != nil { + t.Fatalf("failed to retrieve node data for %x", hash) + } + results = append(results, trie.CodeSyncResult{Hash: hash, Data: data}) + + if len(results) >= cap(results) { + break + } } - if err != nil { - t.Fatalf("failed to retrieve node data for %x", hash) + for _, result := range results { + if err := sched.ProcessCode(result); err != nil { + t.Fatalf("failed to process result %v", err) + } } - results = append(results, trie.SyncResult{Hash: hash, Data: data}) + } + if len(nodeQueue) > 0 { + results := make([]trie.NodeSyncResult, 0, len(nodeQueue)/2+1) + for path, element := range nodeQueue { + delete(nodeQueue, path) - if len(results) >= cap(results) { - break + data, err := srcDb.TrieDB().Node(element.hash) + if err != nil { + t.Fatalf("failed to retrieve node data for %x", element.hash) + } + results = append(results, trie.NodeSyncResult{Path: path, Data: data}) + + if len(results) >= cap(results) { + break + } } - } - // Feed the retrieved results back and queue new tasks - for _, result := range results { - if err := sched.Process(result); err != nil { - t.Fatalf("failed to process result %v", err) + // Feed the retrieved results back and queue new tasks + for _, result := range results { + if err := sched.ProcessNode(result); err != nil { + t.Fatalf("failed to process result %v", err) + } } } batch := dstDb.NewBatch() @@ -384,12 +523,17 @@ func TestIterativeRandomDelayedStateSync(t *testing.T) { t.Fatalf("failed to commit data: %v", err) } batch.Write() - for _, result := range results { - delete(queue, result.Hash) + + paths, nodes, codes := sched.Missing(0) + for i, path := range paths { + nodeQueue[path] = stateElement{ + path: path, + hash: nodes[i], + syncPath: trie.NewSyncPath([]byte(path)), + } } - nodes, _, codes = sched.Missing(0) - for _, hash := range append(nodes, codes...) { - queue[hash] = struct{}{} + for _, hash := range codes { + codeQueue[hash] = struct{}{} } } // Cross check that the two states are in sync @@ -410,34 +554,68 @@ func TestIncompleteStateSync(t *testing.T) { } } isCode[common.BytesToHash(emptyCodeHash)] = struct{}{} - checkTrieConsistency(srcDb.TrieDB().DiskDB().(ethdb.Database), srcRoot) + checkTrieConsistency(srcDb.DiskDB(), srcRoot) // Create a destination state and sync with the scheduler dstDb := rawdb.NewMemoryDatabase() sched := NewStateSync(srcRoot, dstDb, nil) - var added []common.Hash - - nodes, _, codes := sched.Missing(1) - queue := append(append([]common.Hash{}, nodes...), codes...) - - for len(queue) > 0 { + var ( + addedCodes []common.Hash + addedNodes []common.Hash + ) + nodeQueue := make(map[string]stateElement) + codeQueue := make(map[common.Hash]struct{}) + paths, nodes, codes := sched.Missing(1) + for i, path := range paths { + nodeQueue[path] = stateElement{ + path: path, + hash: nodes[i], + syncPath: trie.NewSyncPath([]byte(path)), + } + } + for _, hash := range codes { + codeQueue[hash] = struct{}{} + } + for len(nodeQueue)+len(codeQueue) > 0 { // Fetch a batch of state nodes - results := make([]trie.SyncResult, len(queue)) - for i, hash := range queue { - data, err := srcDb.TrieDB().Node(hash) - if err != nil { - data, err = srcDb.ContractCode(common.Hash{}, hash) + if len(codeQueue) > 0 { + results := make([]trie.CodeSyncResult, 0, len(codeQueue)) + for hash := range codeQueue { + data, err := srcDb.ContractCode(common.Hash{}, hash) + if err != nil { + t.Fatalf("failed to retrieve node data for %x", hash) + } + results = append(results, trie.CodeSyncResult{Hash: hash, Data: data}) + addedCodes = append(addedCodes, hash) } - if err != nil { - t.Fatalf("failed to retrieve node data for %x", hash) + // Process each of the state nodes + for _, result := range results { + if err := sched.ProcessCode(result); err != nil { + t.Fatalf("failed to process result %v", err) + } } - results[i] = trie.SyncResult{Hash: hash, Data: data} } - // Process each of the state nodes - for _, result := range results { - if err := sched.Process(result); err != nil { - t.Fatalf("failed to process result %v", err) + var nodehashes []common.Hash + if len(nodeQueue) > 0 { + results := make([]trie.NodeSyncResult, 0, len(nodeQueue)) + for key, element := range nodeQueue { + data, err := srcDb.TrieDB().Node(element.hash) + if err != nil { + t.Fatalf("failed to retrieve node data for %x", element.hash) + } + results = append(results, trie.NodeSyncResult{Path: key, Data: data}) + + if element.hash != srcRoot { + addedNodes = append(addedNodes, element.hash) + } + nodehashes = append(nodehashes, element.hash) + } + // Process each of the state nodes + for _, result := range results { + if err := sched.ProcessNode(result); err != nil { + t.Fatalf("failed to process result %v", err) + } } } batch := dstDb.NewBatch() @@ -445,43 +623,44 @@ func TestIncompleteStateSync(t *testing.T) { t.Fatalf("failed to commit data: %v", err) } batch.Write() - for _, result := range results { - added = append(added, result.Hash) - // Check that all known sub-tries added so far are complete or missing entirely. - if _, ok := isCode[result.Hash]; ok { - continue - } + + for _, root := range nodehashes { // Can't use checkStateConsistency here because subtrie keys may have odd // length and crash in LeafKey. - if err := checkTrieConsistency(dstDb, result.Hash); err != nil { + if err := checkTrieConsistency(dstDb, root); err != nil { t.Fatalf("state inconsistent: %v", err) } } // Fetch the next batch to retrieve - nodes, _, codes = sched.Missing(1) - queue = append(append(queue[:0], nodes...), codes...) + nodeQueue = make(map[string]stateElement) + codeQueue = make(map[common.Hash]struct{}) + paths, nodes, codes := sched.Missing(1) + for i, path := range paths { + nodeQueue[path] = stateElement{ + path: path, + hash: nodes[i], + syncPath: trie.NewSyncPath([]byte(path)), + } + } + for _, hash := range codes { + codeQueue[hash] = struct{}{} + } } // Sanity check that removing any node from the database is detected - for _, node := range added[1:] { - var ( - key = node.Bytes() - _, code = isCode[node] - val []byte - ) - if code { - val = rawdb.ReadCode(dstDb, node) - rawdb.DeleteCode(dstDb, node) - } else { - val = rawdb.ReadTrieNode(dstDb, node) - rawdb.DeleteTrieNode(dstDb, node) + for _, node := range addedCodes { + val := rawdb.ReadCode(dstDb, node) + rawdb.DeleteCode(dstDb, node) + if err := checkStateConsistency(dstDb, srcRoot); err == nil { + t.Errorf("trie inconsistency not caught, missing: %x", node) } - if err := checkStateConsistency(dstDb, added[0]); err == nil { - t.Fatalf("trie inconsistency not caught, missing: %x", key) - } - if code { - rawdb.WriteCode(dstDb, node, val) - } else { - rawdb.WriteTrieNode(dstDb, node, val) + rawdb.WriteCode(dstDb, node, val) + } + for _, node := range addedNodes { + val := rawdb.ReadTrieNode(dstDb, node) + rawdb.DeleteTrieNode(dstDb, node) + if err := checkStateConsistency(dstDb, srcRoot); err == nil { + t.Errorf("trie inconsistency not caught, missing: %v", node.Hex()) } + rawdb.WriteTrieNode(dstDb, node, val) } } diff --git a/core/state/trie_prefetcher.go b/core/state/trie_prefetcher.go index 25c3730e3f7af..2e16f587ce561 100644 --- a/core/state/trie_prefetcher.go +++ b/core/state/trie_prefetcher.go @@ -25,7 +25,7 @@ import ( ) var ( - // triePrefetchMetricsPrefix is the prefix under which to publis the metrics. + // triePrefetchMetricsPrefix is the prefix under which to publish the metrics. triePrefetchMetricsPrefix = "trie/prefetch/" ) @@ -35,10 +35,10 @@ var ( // // Note, the prefetcher's API is not thread safe. type triePrefetcher struct { - db Database // Database to fetch trie nodes through - root common.Hash // Root hash of theaccount trie for metrics - fetches map[common.Hash]Trie // Partially or fully fetcher tries - fetchers map[common.Hash]*subfetcher // Subfetchers for each trie + db Database // Database to fetch trie nodes through + root common.Hash // Root hash of the account trie for metrics + fetches map[string]Trie // Partially or fully fetcher tries + fetchers map[string]*subfetcher // Subfetchers for each trie deliveryMissMeter metrics.Meter accountLoadMeter metrics.Meter @@ -51,13 +51,12 @@ type triePrefetcher struct { storageWasteMeter metrics.Meter } -// newTriePrefetcher func newTriePrefetcher(db Database, root common.Hash, namespace string) *triePrefetcher { prefix := triePrefetchMetricsPrefix + namespace p := &triePrefetcher{ db: db, root: root, - fetchers: make(map[common.Hash]*subfetcher), // Active prefetchers use the fetchers map + fetchers: make(map[string]*subfetcher), // Active prefetchers use the fetchers map deliveryMissMeter: metrics.GetOrRegisterMeter(prefix+"/deliverymiss", nil), accountLoadMeter: metrics.GetOrRegisterMeter(prefix+"/account/load", nil), @@ -112,7 +111,7 @@ func (p *triePrefetcher) copy() *triePrefetcher { copy := &triePrefetcher{ db: p.db, root: p.root, - fetches: make(map[common.Hash]Trie), // Active prefetchers use the fetches map + fetches: make(map[string]Trie), // Active prefetchers use the fetches map deliveryMissMeter: p.deliveryMissMeter, accountLoadMeter: p.accountLoadMeter, @@ -127,38 +126,43 @@ func (p *triePrefetcher) copy() *triePrefetcher { // If the prefetcher is already a copy, duplicate the data if p.fetches != nil { for root, fetch := range p.fetches { + if fetch == nil { + continue + } copy.fetches[root] = p.db.CopyTrie(fetch) } return copy } // Otherwise we're copying an active fetcher, retrieve the current states - for root, fetcher := range p.fetchers { - copy.fetches[root] = fetcher.peek() + for id, fetcher := range p.fetchers { + copy.fetches[id] = fetcher.peek() } return copy } // prefetch schedules a batch of trie items to prefetch. -func (p *triePrefetcher) prefetch(root common.Hash, keys [][]byte) { +func (p *triePrefetcher) prefetch(owner common.Hash, root common.Hash, keys [][]byte) { // If the prefetcher is an inactive one, bail out if p.fetches != nil { return } // Active fetcher, schedule the retrievals - fetcher := p.fetchers[root] + id := p.trieID(owner, root) + fetcher := p.fetchers[id] if fetcher == nil { - fetcher = newSubfetcher(p.db, root) - p.fetchers[root] = fetcher + fetcher = newSubfetcher(p.db, p.root, owner, root) + p.fetchers[id] = fetcher } fetcher.schedule(keys) } // trie returns the trie matching the root hash, or nil if the prefetcher doesn't // have it. -func (p *triePrefetcher) trie(root common.Hash) Trie { +func (p *triePrefetcher) trie(owner common.Hash, root common.Hash) Trie { // If the prefetcher is inactive, return from existing deep copies + id := p.trieID(owner, root) if p.fetches != nil { - trie := p.fetches[root] + trie := p.fetches[id] if trie == nil { p.deliveryMissMeter.Mark(1) return nil @@ -166,7 +170,7 @@ func (p *triePrefetcher) trie(root common.Hash) Trie { return p.db.CopyTrie(trie) } // Otherwise the prefetcher is active, bail if no trie was prefetched for this root - fetcher := p.fetchers[root] + fetcher := p.fetchers[id] if fetcher == nil { p.deliveryMissMeter.Mark(1) return nil @@ -185,27 +189,34 @@ func (p *triePrefetcher) trie(root common.Hash) Trie { // used marks a batch of state items used to allow creating statistics as to // how useful or wasteful the prefetcher is. -func (p *triePrefetcher) used(root common.Hash, used [][]byte) { - if fetcher := p.fetchers[root]; fetcher != nil { +func (p *triePrefetcher) used(owner common.Hash, root common.Hash, used [][]byte) { + if fetcher := p.fetchers[p.trieID(owner, root)]; fetcher != nil { fetcher.used = used } } +// trieID returns an unique trie identifier consists the trie owner and root hash. +func (p *triePrefetcher) trieID(owner common.Hash, root common.Hash) string { + return string(append(owner.Bytes(), root.Bytes()...)) +} + // subfetcher is a trie fetcher goroutine responsible for pulling entries for a // single trie. It is spawned when a new root is encountered and lives until the // main prefetcher is paused and either all requested items are processed or if // the trie being worked on is retrieved from the prefetcher. type subfetcher struct { - db Database // Database to load trie nodes through - root common.Hash // Root hash of the trie to prefetch - trie Trie // Trie being populated with nodes + db Database // Database to load trie nodes through + state common.Hash // Root hash of the state to prefetch + owner common.Hash // Owner of the trie, usually account hash + root common.Hash // Root hash of the trie to prefetch + trie Trie // Trie being populated with nodes tasks [][]byte // Items queued up for retrieval lock sync.Mutex // Lock protecting the task queue wake chan struct{} // Wake channel if a new task is scheduled stop chan struct{} // Channel to interrupt processing - term chan struct{} // Channel to signal iterruption + term chan struct{} // Channel to signal interruption copy chan chan Trie // Channel to request a copy of the current trie seen map[string]struct{} // Tracks the entries already loaded @@ -215,15 +226,17 @@ type subfetcher struct { // newSubfetcher creates a goroutine to prefetch state items belonging to a // particular root hash. -func newSubfetcher(db Database, root common.Hash) *subfetcher { +func newSubfetcher(db Database, state common.Hash, owner common.Hash, root common.Hash) *subfetcher { sf := &subfetcher{ - db: db, - root: root, - wake: make(chan struct{}, 1), - stop: make(chan struct{}), - term: make(chan struct{}), - copy: make(chan chan Trie), - seen: make(map[string]struct{}), + db: db, + state: state, + owner: owner, + root: root, + wake: make(chan struct{}, 1), + stop: make(chan struct{}), + term: make(chan struct{}), + copy: make(chan chan Trie), + seen: make(map[string]struct{}), } go sf.loop() return sf @@ -279,13 +292,21 @@ func (sf *subfetcher) loop() { defer close(sf.term) // Start by opening the trie and stop processing if it fails - trie, err := sf.db.OpenTrie(sf.root) - if err != nil { - log.Warn("Trie prefetcher failed opening trie", "root", sf.root, "err", err) - return + if sf.owner == (common.Hash{}) { + trie, err := sf.db.OpenTrie(sf.root) + if err != nil { + log.Warn("Trie prefetcher failed opening trie", "root", sf.root, "err", err) + return + } + sf.trie = trie + } else { + trie, err := sf.db.OpenStorageTrie(sf.state, sf.owner, sf.root) + if err != nil { + log.Warn("Trie prefetcher failed opening trie", "root", sf.root, "err", err) + return + } + sf.trie = trie } - sf.trie = trie - // Trie opened successfully, keep prefetching items for { select { @@ -315,7 +336,11 @@ func (sf *subfetcher) loop() { if _, ok := sf.seen[string(task)]; ok { sf.dups++ } else { - sf.trie.TryGet(task) + if len(task) == len(common.Address{}) { + sf.trie.TryGetAccount(task) + } else { + sf.trie.TryGet(task) + } sf.seen[string(task)] = struct{}{} } } diff --git a/core/state/trie_prefetcher_test.go b/core/state/trie_prefetcher_test.go index 35dc7a2c0da47..cb0b67d7ea79f 100644 --- a/core/state/trie_prefetcher_test.go +++ b/core/state/trie_prefetcher_test.go @@ -47,20 +47,20 @@ func TestCopyAndClose(t *testing.T) { db := filledStateDB() prefetcher := newTriePrefetcher(db.db, db.originalRoot, "") skey := common.HexToHash("aaa") - prefetcher.prefetch(db.originalRoot, [][]byte{skey.Bytes()}) - prefetcher.prefetch(db.originalRoot, [][]byte{skey.Bytes()}) + prefetcher.prefetch(common.Hash{}, db.originalRoot, [][]byte{skey.Bytes()}) + prefetcher.prefetch(common.Hash{}, db.originalRoot, [][]byte{skey.Bytes()}) time.Sleep(1 * time.Second) - a := prefetcher.trie(db.originalRoot) - prefetcher.prefetch(db.originalRoot, [][]byte{skey.Bytes()}) - b := prefetcher.trie(db.originalRoot) + a := prefetcher.trie(common.Hash{}, db.originalRoot) + prefetcher.prefetch(common.Hash{}, db.originalRoot, [][]byte{skey.Bytes()}) + b := prefetcher.trie(common.Hash{}, db.originalRoot) cpy := prefetcher.copy() - cpy.prefetch(db.originalRoot, [][]byte{skey.Bytes()}) - cpy.prefetch(db.originalRoot, [][]byte{skey.Bytes()}) - c := cpy.trie(db.originalRoot) + cpy.prefetch(common.Hash{}, db.originalRoot, [][]byte{skey.Bytes()}) + cpy.prefetch(common.Hash{}, db.originalRoot, [][]byte{skey.Bytes()}) + c := cpy.trie(common.Hash{}, db.originalRoot) prefetcher.close() cpy2 := cpy.copy() - cpy2.prefetch(db.originalRoot, [][]byte{skey.Bytes()}) - d := cpy2.trie(db.originalRoot) + cpy2.prefetch(common.Hash{}, db.originalRoot, [][]byte{skey.Bytes()}) + d := cpy2.trie(common.Hash{}, db.originalRoot) cpy.close() cpy2.close() if a.Hash() != b.Hash() || a.Hash() != c.Hash() || a.Hash() != d.Hash() { @@ -72,10 +72,10 @@ func TestUseAfterClose(t *testing.T) { db := filledStateDB() prefetcher := newTriePrefetcher(db.db, db.originalRoot, "") skey := common.HexToHash("aaa") - prefetcher.prefetch(db.originalRoot, [][]byte{skey.Bytes()}) - a := prefetcher.trie(db.originalRoot) + prefetcher.prefetch(common.Hash{}, db.originalRoot, [][]byte{skey.Bytes()}) + a := prefetcher.trie(common.Hash{}, db.originalRoot) prefetcher.close() - b := prefetcher.trie(db.originalRoot) + b := prefetcher.trie(common.Hash{}, db.originalRoot) if a == nil { t.Fatal("Prefetching before close should not return nil") } @@ -88,13 +88,13 @@ func TestCopyClose(t *testing.T) { db := filledStateDB() prefetcher := newTriePrefetcher(db.db, db.originalRoot, "") skey := common.HexToHash("aaa") - prefetcher.prefetch(db.originalRoot, [][]byte{skey.Bytes()}) + prefetcher.prefetch(common.Hash{}, db.originalRoot, [][]byte{skey.Bytes()}) cpy := prefetcher.copy() - a := prefetcher.trie(db.originalRoot) - b := cpy.trie(db.originalRoot) + a := prefetcher.trie(common.Hash{}, db.originalRoot) + b := cpy.trie(common.Hash{}, db.originalRoot) prefetcher.close() - c := prefetcher.trie(db.originalRoot) - d := cpy.trie(db.originalRoot) + c := prefetcher.trie(common.Hash{}, db.originalRoot) + d := cpy.trie(common.Hash{}, db.originalRoot) if a == nil { t.Fatal("Prefetching before close should not return nil") } diff --git a/core/state_processor.go b/core/state_processor.go index d4c77ae410421..e511697c5f6af 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -79,7 +79,7 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg return nil, nil, 0, fmt.Errorf("could not apply tx %d [%v]: %w", i, tx.Hash().Hex(), err) } statedb.Prepare(tx.Hash(), i) - receipt, err := applyTransaction(msg, p.config, p.bc, nil, gp, statedb, blockNumber, blockHash, tx, usedGas, vmenv) + receipt, err := applyTransaction(msg, p.config, nil, gp, statedb, blockNumber, blockHash, tx, usedGas, vmenv) if err != nil { return nil, nil, 0, fmt.Errorf("could not apply tx %d [%v]: %w", i, tx.Hash().Hex(), err) } @@ -92,7 +92,7 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg return receipts, allLogs, *usedGas, nil } -func applyTransaction(msg types.Message, config *params.ChainConfig, bc ChainContext, author *common.Address, gp *GasPool, statedb *state.StateDB, blockNumber *big.Int, blockHash common.Hash, tx *types.Transaction, usedGas *uint64, evm *vm.EVM) (*types.Receipt, error) { +func applyTransaction(msg types.Message, config *params.ChainConfig, author *common.Address, gp *GasPool, statedb *state.StateDB, blockNumber *big.Int, blockHash common.Hash, tx *types.Transaction, usedGas *uint64, evm *vm.EVM) (*types.Receipt, error) { // Create a new context to be used in the EVM environment. txContext := NewEVMTxContext(msg) evm.Reset(txContext, statedb) @@ -149,5 +149,5 @@ func ApplyTransaction(config *params.ChainConfig, bc ChainContext, author *commo // Create a new context to be used in the EVM environment blockContext := NewEVMBlockContext(header, bc, author) vmenv := vm.NewEVM(blockContext, vm.TxContext{}, statedb, config, cfg) - return applyTransaction(msg, config, bc, author, gp, statedb, header.Number, header.Hash(), tx, usedGas, vmenv) + return applyTransaction(msg, config, author, gp, statedb, header.Number, header.Hash(), tx, usedGas, vmenv) } diff --git a/core/state_processor_test.go b/core/state_processor_test.go index aa8e4bebf9d49..1df4a0e804189 100644 --- a/core/state_processor_test.go +++ b/core/state_processor_test.go @@ -91,8 +91,7 @@ func TestStateProcessorErrors(t *testing.T) { }, }, } - genesis = gspec.MustCommit(db) - blockchain, _ = NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) + blockchain, _ = NewBlockChain(db, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) ) defer blockchain.Stop() bigNumber := new(big.Int).SetBytes(common.FromHex("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")) @@ -197,7 +196,7 @@ func TestStateProcessorErrors(t *testing.T) { want: "could not apply tx 0 [0xd82a0c2519acfeac9a948258c47e784acd20651d9d80f9a1c67b4137651c3a24]: insufficient funds for gas * price + value: address 0x71562b71999873DB5b286dF957af199Ec94617F7 have 1000000000000000000 want 2431633873983640103894990685182446064918669677978451844828609264166175722438635000", }, } { - block := GenerateBadBlock(genesis, ethash.NewFaker(), tt.txs, gspec.Config) + block := GenerateBadBlock(gspec.ToBlock(), ethash.NewFaker(), tt.txs, gspec.Config) _, err := blockchain.InsertChain(types.Blocks{block}) if err == nil { t.Fatal("block imported without errors") @@ -232,8 +231,7 @@ func TestStateProcessorErrors(t *testing.T) { }, }, } - genesis = gspec.MustCommit(db) - blockchain, _ = NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) + blockchain, _ = NewBlockChain(db, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) ) defer blockchain.Stop() for i, tt := range []struct { @@ -247,7 +245,7 @@ func TestStateProcessorErrors(t *testing.T) { want: "could not apply tx 0 [0x88626ac0d53cb65308f2416103c62bb1f18b805573d4f96a3640bbbfff13c14f]: transaction type not supported", }, } { - block := GenerateBadBlock(genesis, ethash.NewFaker(), tt.txs, gspec.Config) + block := GenerateBadBlock(gspec.ToBlock(), ethash.NewFaker(), tt.txs, gspec.Config) _, err := blockchain.InsertChain(types.Blocks{block}) if err == nil { t.Fatal("block imported without errors") @@ -272,8 +270,7 @@ func TestStateProcessorErrors(t *testing.T) { }, }, } - genesis = gspec.MustCommit(db) - blockchain, _ = NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) + blockchain, _ = NewBlockChain(db, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) ) defer blockchain.Stop() for i, tt := range []struct { @@ -287,7 +284,7 @@ func TestStateProcessorErrors(t *testing.T) { want: "could not apply tx 0 [0x88626ac0d53cb65308f2416103c62bb1f18b805573d4f96a3640bbbfff13c14f]: sender not an eoa: address 0x71562b71999873DB5b286dF957af199Ec94617F7, codehash: 0x9280914443471259d4570a8661015ae4a5b80186dbc619658fb494bebc3da3d1", }, } { - block := GenerateBadBlock(genesis, ethash.NewFaker(), tt.txs, gspec.Config) + block := GenerateBadBlock(gspec.ToBlock(), ethash.NewFaker(), tt.txs, gspec.Config) _, err := blockchain.InsertChain(types.Blocks{block}) if err == nil { t.Fatal("block imported without errors") diff --git a/core/state_transition.go b/core/state_transition.go index 3b5f81b166329..4048c02507a9e 100644 --- a/core/state_transition.go +++ b/core/state_transition.go @@ -31,23 +31,26 @@ import ( var emptyCodeHash = crypto.Keccak256Hash(nil) -/* -The State Transitioning Model - -A state transition is a change made when a transaction is applied to the current world state -The state transitioning model does all the necessary work to work out a valid new state root. - -1) Nonce handling -2) Pre pay gas -3) Create a new state object if the recipient is \0*32 -4) Value transfer -== If contract creation == - 4a) Attempt to run transaction data - 4b) If valid, use result as code for the new state object -== end == -5) Run Script section -6) Derive new state root -*/ +// The State Transitioning Model +// +// A state transition is a change made when a transaction is applied to the current world +// state. The state transitioning model does all the necessary work to work out a valid new +// state root. +// +// 1. Nonce handling +// 2. Pre pay gas +// 3. Create a new state object if the recipient is \0*32 +// 4. Value transfer +// +// == If contract creation == +// +// 4a. Attempt to run transaction data +// 4b. If valid, use result as code for the new state object +// +// == end == +// +// 5. Run Script section +// 6. Derive new state root type StateTransition struct { gp *GasPool msg Message @@ -262,13 +265,10 @@ func (st *StateTransition) preCheck() error { // TransitionDb will transition the state by applying the current message and // returning the evm execution result with following fields. // -// - used gas: -// total gas used (including gas being refunded) -// - returndata: -// the returned data from evm -// - concrete execution error: -// various **EVM** error which aborts the execution, -// e.g. ErrOutOfGas, ErrExecutionReverted +// - used gas: total gas used (including gas being refunded) +// - returndata: the returned data from evm +// - concrete execution error: various EVM errors which abort the execution, e.g. +// ErrOutOfGas, ErrExecutionReverted // // However if any consensus issue encountered, return the error directly with // nil evm execution result. @@ -344,7 +344,16 @@ func (st *StateTransition) TransitionDb() (*ExecutionResult, error) { if rules.IsLondon { effectiveTip = cmath.BigMin(st.gasTipCap, new(big.Int).Sub(st.gasFeeCap, st.evm.Context.BaseFee)) } - st.state.AddBalance(st.evm.Context.Coinbase, new(big.Int).Mul(new(big.Int).SetUint64(st.gasUsed()), effectiveTip)) + + if st.evm.Config.NoBaseFee && st.gasFeeCap.Sign() == 0 && st.gasTipCap.Sign() == 0 { + // Skip fee payment when NoBaseFee is set and the fee fields + // are 0. This avoids a negative effectiveTip being applied to + // the coinbase when simulating calls. + } else { + fee := new(big.Int).SetUint64(st.gasUsed()) + fee.Mul(fee, effectiveTip) + st.state.AddBalance(st.evm.Context.Coinbase, fee) + } return &ExecutionResult{ UsedGas: st.gasUsed(), diff --git a/core/tx_journal.go b/core/tx_journal.go index 5453ee191658d..62344f5646762 100644 --- a/core/tx_journal.go +++ b/core/tx_journal.go @@ -19,6 +19,7 @@ package core import ( "errors" "io" + "io/fs" "os" "github.com/ethereum/go-ethereum/common" @@ -57,12 +58,12 @@ func newTxJournal(path string) *txJournal { // load parses a transaction journal dump from disk, loading its contents into // the specified pool. func (journal *txJournal) load(add func([]*types.Transaction) []error) error { - // Skip the parsing if the journal file doesn't exist at all - if !common.FileExist(journal.path) { - return nil - } // Open the journal for loading any past transactions input, err := os.Open(journal.path) + if errors.Is(err, fs.ErrNotExist) { + // Skip the parsing if the journal file doesn't exist at all + return nil + } if err != nil { return err } diff --git a/core/tx_noncer.go b/core/tx_noncer.go index d6d220077507a..257beffa06c65 100644 --- a/core/tx_noncer.go +++ b/core/tx_noncer.go @@ -49,7 +49,9 @@ func (txn *txNoncer) get(addr common.Address) uint64 { defer txn.lock.Unlock() if _, ok := txn.nonces[addr]; !ok { - txn.nonces[addr] = txn.fallback.GetNonce(addr) + if nonce := txn.fallback.GetNonce(addr); nonce != 0 { + txn.nonces[addr] = nonce + } } return txn.nonces[addr] } @@ -64,13 +66,15 @@ func (txn *txNoncer) set(addr common.Address, nonce uint64) { } // setIfLower updates a new virtual nonce into the virtual state database if the -// the new one is lower. +// new one is lower. func (txn *txNoncer) setIfLower(addr common.Address, nonce uint64) { txn.lock.Lock() defer txn.lock.Unlock() if _, ok := txn.nonces[addr]; !ok { - txn.nonces[addr] = txn.fallback.GetNonce(addr) + if nonce := txn.fallback.GetNonce(addr); nonce != 0 { + txn.nonces[addr] = nonce + } } if txn.nonces[addr] <= nonce { return diff --git a/core/tx_pool.go b/core/tx_pool.go index 81a726bae4abc..ee8b9f7a43f08 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -498,11 +498,11 @@ func (pool *TxPool) Content() (map[common.Address]types.Transactions, map[common pool.mu.Lock() defer pool.mu.Unlock() - pending := make(map[common.Address]types.Transactions) + pending := make(map[common.Address]types.Transactions, len(pool.pending)) for addr, list := range pool.pending { pending[addr] = list.Flatten() } - queued := make(map[common.Address]types.Transactions) + queued := make(map[common.Address]types.Transactions, len(pool.queue)) for addr, list := range pool.queue { queued[addr] = list.Flatten() } @@ -1459,7 +1459,7 @@ func (pool *TxPool) truncatePending() { pendingRateLimitMeter.Mark(int64(pendingBeforeCap - pending)) } -// truncateQueue drops the oldes transactions in the queue if the pool is above the global queue limit. +// truncateQueue drops the oldest transactions in the queue if the pool is above the global queue limit. func (pool *TxPool) truncateQueue() { queued := uint64(0) for _, list := range pool.queue { @@ -1588,7 +1588,7 @@ type accountSet struct { // derivations. func newAccountSet(signer types.Signer, addrs ...common.Address) *accountSet { as := &accountSet{ - accounts: make(map[common.Address]struct{}), + accounts: make(map[common.Address]struct{}, len(addrs)), signer: signer, } for _, addr := range addrs { @@ -1603,10 +1603,6 @@ func (as *accountSet) contains(addr common.Address) bool { return exist } -func (as *accountSet) empty() bool { - return len(as.accounts) == 0 -} - // containsTx checks if the sender of a given tx is within the set. If the sender // cannot be derived, this method returns false. func (as *accountSet) containsTx(tx *types.Transaction) bool { diff --git a/core/tx_pool_test.go b/core/tx_pool_test.go index dd2407470daa5..2fd0f529f8f2c 100644 --- a/core/tx_pool_test.go +++ b/core/tx_pool_test.go @@ -669,7 +669,6 @@ func TestTransactionPostponing(t *testing.T) { // Add a batch consecutive pending transactions for validation txs := []*types.Transaction{} for i, key := range keys { - for j := 0; j < 100; j++ { var tx *types.Transaction if (i+j)%2 == 0 { diff --git a/core/types/block.go b/core/types/block.go index 8386c4c440e81..8942082b6e487 100644 --- a/core/types/block.go +++ b/core/types/block.go @@ -117,7 +117,11 @@ var headerSize = common.StorageSize(reflect.TypeOf(Header{}).Size()) // Size returns the approximate memory used by all internal contents. It is used // to approximate and limit the memory consumption of various caches. func (h *Header) Size() common.StorageSize { - return headerSize + common.StorageSize(len(h.Extra)+(h.Difficulty.BitLen()+h.Number.BitLen())/8) + var baseFeeBits int + if h.BaseFee != nil { + baseFeeBits = h.BaseFee.BitLen() + } + return headerSize + common.StorageSize(len(h.Extra)+(h.Difficulty.BitLen()+h.Number.BitLen()+baseFeeBits)/8) } // SanityCheck checks a few basic things -- these checks are way beyond what @@ -172,10 +176,6 @@ type Block struct { hash atomic.Value size atomic.Value - // Td is used by package core to store the total difficulty - // of the chain up to and including the block. - td *big.Int - // These fields are used by package eth to track // inter-peer block relay. ReceivedAt time.Time @@ -197,7 +197,7 @@ type extblock struct { // are ignored and set to values derived from the given txs, uncles // and receipts. func NewBlock(header *Header, txs []*Transaction, uncles []*Header, receipts []*Receipt, hasher TrieHasher) *Block { - b := &Block{header: CopyHeader(header), td: new(big.Int)} + b := &Block{header: CopyHeader(header)} // TODO: panic if len(txs) != len(receipts) if len(txs) == 0 { @@ -321,7 +321,7 @@ func (b *Block) Header() *Header { return CopyHeader(b.header) } func (b *Block) Body() *Body { return &Body{b.transactions, b.uncles} } // Size returns the true RLP encoded storage size of the block, either by encoding -// and returning it, or returning a previsouly cached value. +// and returning it, or returning a previously cached value. func (b *Block) Size() common.StorageSize { if size := b.size.Load(); size != nil { return size.(common.StorageSize) diff --git a/core/types/block_test.go b/core/types/block_test.go index aa1db2f4faade..9e7f581b1dc4d 100644 --- a/core/types/block_test.go +++ b/core/types/block_test.go @@ -314,7 +314,7 @@ func TestRlpDecodeParentHash(t *testing.T) { } // Also test a very very large header. { - // The rlp-encoding of the heder belowCauses _total_ length of 65540, + // The rlp-encoding of the header belowCauses _total_ length of 65540, // which is the first to blow the fast-path. h := &Header{ ParentHash: want, diff --git a/core/types/bloom9.go b/core/types/bloom9.go index 1793c2adc73c9..a560a20724fd7 100644 --- a/core/types/bloom9.go +++ b/core/types/bloom9.go @@ -154,7 +154,7 @@ func bloomValues(data []byte, hashbuf []byte) (uint, byte, uint, byte, uint, byt return i1, v1, i2, v2, i3, v3 } -// BloomLookup is a convenience-method to check presence int he bloom filter +// BloomLookup is a convenience-method to check presence in the bloom filter func BloomLookup(bin Bloom, topic bytesBacked) bool { return bin.Test(topic.Bytes()) } diff --git a/core/types/bloom9_test.go b/core/types/bloom9_test.go index 893df486dd1bb..d3178d112efb9 100644 --- a/core/types/bloom9_test.go +++ b/core/types/bloom9_test.go @@ -92,7 +92,6 @@ func BenchmarkBloom9Lookup(b *testing.B) { } func BenchmarkCreateBloom(b *testing.B) { - var txs = Transactions{ NewContractCreation(1, big.NewInt(1), 1, big.NewInt(1), nil), NewTransaction(2, common.HexToAddress("0x2"), big.NewInt(2), 2, big.NewInt(2), nil), diff --git a/core/types/hashing.go b/core/types/hashing.go index a115a8842ec3b..3df75432a4b43 100644 --- a/core/types/hashing.go +++ b/core/types/hashing.go @@ -31,7 +31,7 @@ var hasherPool = sync.Pool{ New: func() interface{} { return sha3.NewLegacyKeccak256() }, } -// deriveBufferPool holds temporary encoder buffers for DeriveSha and TX encoding. +// encodeBufferPool holds temporary encoder buffers for DeriveSha and TX encoding. var encodeBufferPool = sync.Pool{ New: func() interface{} { return new(bytes.Buffer) }, } diff --git a/core/types/hashing_test.go b/core/types/hashing_test.go index de71ee41a47d7..294a3977d03b0 100644 --- a/core/types/hashing_test.go +++ b/core/types/hashing_test.go @@ -39,8 +39,7 @@ func TestDeriveSha(t *testing.T) { t.Fatal(err) } for len(txs) < 1000 { - tr, _ := trie.New(common.Hash{}, trie.NewDatabase(rawdb.NewMemoryDatabase())) - exp := types.DeriveSha(txs, tr) + exp := types.DeriveSha(txs, trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase()))) got := types.DeriveSha(txs, trie.NewStackTrie(nil)) if !bytes.Equal(got[:], exp[:]) { t.Fatalf("%d txs: got %x exp %x", len(txs), got, exp) @@ -87,8 +86,7 @@ func BenchmarkDeriveSha200(b *testing.B) { b.ResetTimer() b.ReportAllocs() for i := 0; i < b.N; i++ { - tr, _ := trie.New(common.Hash{}, trie.NewDatabase(rawdb.NewMemoryDatabase())) - exp = types.DeriveSha(txs, tr) + exp = types.DeriveSha(txs, trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase()))) } }) @@ -109,8 +107,7 @@ func TestFuzzDeriveSha(t *testing.T) { rndSeed := mrand.Int() for i := 0; i < 10; i++ { seed := rndSeed + i - tr, _ := trie.New(common.Hash{}, trie.NewDatabase(rawdb.NewMemoryDatabase())) - exp := types.DeriveSha(newDummy(i), tr) + exp := types.DeriveSha(newDummy(i), trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase()))) got := types.DeriveSha(newDummy(i), trie.NewStackTrie(nil)) if !bytes.Equal(got[:], exp[:]) { printList(newDummy(seed)) @@ -138,8 +135,7 @@ func TestDerivableList(t *testing.T) { }, } for i, tc := range tcs[1:] { - tr, _ := trie.New(common.Hash{}, trie.NewDatabase(rawdb.NewMemoryDatabase())) - exp := types.DeriveSha(flatList(tc), tr) + exp := types.DeriveSha(flatList(tc), trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase()))) got := types.DeriveSha(flatList(tc), trie.NewStackTrie(nil)) if !bytes.Equal(got[:], exp[:]) { t.Fatalf("case %d: got %x exp %x", i, got, exp) @@ -201,7 +197,7 @@ func printList(l types.DerivableList) { for i := 0; i < l.Len(); i++ { var buf bytes.Buffer l.EncodeIndex(i, &buf) - fmt.Printf("\"0x%x\",\n", buf.Bytes()) + fmt.Printf("\"%#x\",\n", buf.Bytes()) } fmt.Printf("},\n") } diff --git a/core/types/log.go b/core/types/log.go index ee323ba868041..eb30957b12788 100644 --- a/core/types/log.go +++ b/core/types/log.go @@ -98,8 +98,8 @@ func (l *Log) DecodeRLP(s *rlp.Stream) error { return err } -// LogForStorage is a wrapper around a Log that flattens and parses the entire content of -// a log including non-consensus fields. +// LogForStorage is a wrapper around a Log that handles +// backward compatibility with prior storage formats. type LogForStorage Log // EncodeRLP implements rlp.Encoder. diff --git a/core/types/receipt.go b/core/types/receipt.go index a913cd0e83be9..bdf48451473c3 100644 --- a/core/types/receipt.go +++ b/core/types/receipt.go @@ -267,8 +267,8 @@ func (r *Receipt) Size() common.StorageSize { return size } -// ReceiptForStorage is a wrapper around a Receipt that flattens and parses the -// entire content of a receipt, as opposed to only the consensus fields originally. +// ReceiptForStorage is a wrapper around a Receipt with RLP serialization +// that omits the Bloom field and deserialization that re-computes it. type ReceiptForStorage Receipt // EncodeRLP implements rlp.Encoder, and flattens all content fields of a receipt diff --git a/core/types/transaction_signing_test.go b/core/types/transaction_signing_test.go index 689fc38a9b660..1c775f129d654 100644 --- a/core/types/transaction_signing_test.go +++ b/core/types/transaction_signing_test.go @@ -111,7 +111,6 @@ func TestEIP155SigningVitalik(t *testing.T) { if from != addr { t.Errorf("%d: expected %x got %x", i, addr, from) } - } } diff --git a/core/types/transaction_test.go b/core/types/transaction_test.go index 2e418b2309861..67e5b3cce3f5b 100644 --- a/core/types/transaction_test.go +++ b/core/types/transaction_test.go @@ -114,7 +114,6 @@ func TestEIP2718TransactionSigHash(t *testing.T) { // This test checks signature operations on access list transactions. func TestEIP2930Signer(t *testing.T) { - var ( key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") keyAddr = crypto.PubkeyToAddress(key.PublicKey) diff --git a/core/vm/contracts.go b/core/vm/contracts.go index 79f1a3611680f..44aa930d47a35 100644 --- a/core/vm/contracts.go +++ b/core/vm/contracts.go @@ -29,8 +29,6 @@ import ( "github.com/ethereum/go-ethereum/crypto/bls12381" "github.com/ethereum/go-ethereum/crypto/bn256" "github.com/ethereum/go-ethereum/params" - - //lint:ignore SA1019 Needed for precompile "golang.org/x/crypto/ripemd160" ) @@ -237,7 +235,7 @@ func (c *dataCopy) RequiredGas(input []byte) uint64 { return uint64(len(input)+31)/32*params.IdentityPerWordGas + params.IdentityBaseGas } func (c *dataCopy) Run(in []byte) ([]byte, error) { - return in, nil + return common.CopyBytes(in), nil } // bigModExp implements a native big integer exponential modular operation. @@ -265,10 +263,10 @@ var ( // modexpMultComplexity implements bigModexp multComplexity formula, as defined in EIP-198 // -// def mult_complexity(x): -// if x <= 64: return x ** 2 -// elif x <= 1024: return x ** 2 // 4 + 96 * x - 3072 -// else: return x ** 2 // 16 + 480 * x - 199680 +// def mult_complexity(x): +// if x <= 64: return x ** 2 +// elif x <= 1024: return x ** 2 // 4 + 96 * x - 3072 +// else: return x ** 2 // 16 + 480 * x - 199680 // // where is x is max(length_of_MODULUS, length_of_BASE) func modexpMultComplexity(x *big.Int) *big.Int { diff --git a/core/vm/evm.go b/core/vm/evm.go index dd55618bf8125..888f4812a5901 100644 --- a/core/vm/evm.go +++ b/core/vm/evm.go @@ -75,7 +75,7 @@ type BlockContext struct { Time *big.Int // Provides information for TIME Difficulty *big.Int // Provides information for DIFFICULTY BaseFee *big.Int // Provides information for BASEFEE - Random *common.Hash // Provides information for RANDOM + Random *common.Hash // Provides information for PREVRANDAO } // TxContext provides the EVM with information about a transaction. diff --git a/core/vm/gas_table.go b/core/vm/gas_table.go index 4c2cb3e5cf79b..57fb1a8d98b2f 100644 --- a/core/vm/gas_table.go +++ b/core/vm/gas_table.go @@ -117,20 +117,21 @@ func gasSStore(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySi return params.SstoreResetGas, nil } } + // The new gas metering is based on net gas costs (EIP-1283): // - // 1. If current value equals new value (this is a no-op), 200 gas is deducted. - // 2. If current value does not equal new value - // 2.1. If original value equals current value (this storage slot has not been changed by the current execution context) - // 2.1.1. If original value is 0, 20000 gas is deducted. - // 2.1.2. Otherwise, 5000 gas is deducted. If new value is 0, add 15000 gas to refund counter. - // 2.2. If original value does not equal current value (this storage slot is dirty), 200 gas is deducted. Apply both of the following clauses. - // 2.2.1. If original value is not 0 - // 2.2.1.1. If current value is 0 (also means that new value is not 0), remove 15000 gas from refund counter. We can prove that refund counter will never go below 0. - // 2.2.1.2. If new value is 0 (also means that current value is not 0), add 15000 gas to refund counter. - // 2.2.2. If original value equals new value (this storage slot is reset) - // 2.2.2.1. If original value is 0, add 19800 gas to refund counter. - // 2.2.2.2. Otherwise, add 4800 gas to refund counter. + // (1.) If current value equals new value (this is a no-op), 200 gas is deducted. + // (2.) If current value does not equal new value + // (2.1.) If original value equals current value (this storage slot has not been changed by the current execution context) + // (2.1.1.) If original value is 0, 20000 gas is deducted. + // (2.1.2.) Otherwise, 5000 gas is deducted. If new value is 0, add 15000 gas to refund counter. + // (2.2.) If original value does not equal current value (this storage slot is dirty), 200 gas is deducted. Apply both of the following clauses. + // (2.2.1.) If original value is not 0 + // (2.2.1.1.) If current value is 0 (also means that new value is not 0), remove 15000 gas from refund counter. We can prove that refund counter will never go below 0. + // (2.2.1.2.) If new value is 0 (also means that current value is not 0), add 15000 gas to refund counter. + // (2.2.2.) If original value equals new value (this storage slot is reset) + // (2.2.2.1.) If original value is 0, add 19800 gas to refund counter. + // (2.2.2.2.) Otherwise, add 4800 gas to refund counter. value := common.Hash(y.Bytes32()) if current == value { // noop (1) return params.NetSstoreNoopGas, nil @@ -162,19 +163,21 @@ func gasSStore(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySi return params.NetSstoreDirtyGas, nil } -// 0. If *gasleft* is less than or equal to 2300, fail the current call. -// 1. If current value equals new value (this is a no-op), SLOAD_GAS is deducted. -// 2. If current value does not equal new value: -// 2.1. If original value equals current value (this storage slot has not been changed by the current execution context): -// 2.1.1. If original value is 0, SSTORE_SET_GAS (20K) gas is deducted. -// 2.1.2. Otherwise, SSTORE_RESET_GAS gas is deducted. If new value is 0, add SSTORE_CLEARS_SCHEDULE to refund counter. -// 2.2. If original value does not equal current value (this storage slot is dirty), SLOAD_GAS gas is deducted. Apply both of the following clauses: -// 2.2.1. If original value is not 0: -// 2.2.1.1. If current value is 0 (also means that new value is not 0), subtract SSTORE_CLEARS_SCHEDULE gas from refund counter. -// 2.2.1.2. If new value is 0 (also means that current value is not 0), add SSTORE_CLEARS_SCHEDULE gas to refund counter. -// 2.2.2. If original value equals new value (this storage slot is reset): -// 2.2.2.1. If original value is 0, add SSTORE_SET_GAS - SLOAD_GAS to refund counter. -// 2.2.2.2. Otherwise, add SSTORE_RESET_GAS - SLOAD_GAS gas to refund counter. +// Here come the EIP220 rules: +// +// (0.) If *gasleft* is less than or equal to 2300, fail the current call. +// (1.) If current value equals new value (this is a no-op), SLOAD_GAS is deducted. +// (2.) If current value does not equal new value: +// (2.1.) If original value equals current value (this storage slot has not been changed by the current execution context): +// (2.1.1.) If original value is 0, SSTORE_SET_GAS (20K) gas is deducted. +// (2.1.2.) Otherwise, SSTORE_RESET_GAS gas is deducted. If new value is 0, add SSTORE_CLEARS_SCHEDULE to refund counter. +// (2.2.) If original value does not equal current value (this storage slot is dirty), SLOAD_GAS gas is deducted. Apply both of the following clauses: +// (2.2.1.) If original value is not 0: +// (2.2.1.1.) If current value is 0 (also means that new value is not 0), subtract SSTORE_CLEARS_SCHEDULE gas from refund counter. +// (2.2.1.2.) If new value is 0 (also means that current value is not 0), add SSTORE_CLEARS_SCHEDULE gas to refund counter. +// (2.2.2.) If original value equals new value (this storage slot is reset): +// (2.2.2.1.) If original value is 0, add SSTORE_SET_GAS - SLOAD_GAS to refund counter. +// (2.2.2.2.) Otherwise, add SSTORE_RESET_GAS - SLOAD_GAS gas to refund counter. func gasSStoreEIP2200(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (uint64, error) { // If we fail the minimum gas availability invariant, fail (0) if contract.Gas <= params.SstoreSentryGasEIP2200 { diff --git a/core/vm/instructions.go b/core/vm/instructions.go index 92be3bf259a3d..22d459233b3dc 100644 --- a/core/vm/instructions.go +++ b/core/vm/instructions.go @@ -21,9 +21,9 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/params" "github.com/holiman/uint256" - "golang.org/x/crypto/sha3" ) func opAdd(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { @@ -238,7 +238,7 @@ func opKeccak256(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ( data := scope.Memory.GetPtr(int64(offset.Uint64()), int64(size.Uint64())) if interpreter.hasher == nil { - interpreter.hasher = sha3.NewLegacyKeccak256().(keccakState) + interpreter.hasher = crypto.NewKeccakState() } else { interpreter.hasher.Reset() } @@ -392,29 +392,29 @@ func opExtCodeCopy(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) // opExtCodeHash returns the code hash of a specified account. // There are several cases when the function is called, while we can relay everything // to `state.GetCodeHash` function to ensure the correctness. -// (1) Caller tries to get the code hash of a normal contract account, state -// should return the relative code hash and set it as the result. // -// (2) Caller tries to get the code hash of a non-existent account, state should -// return common.Hash{} and zero will be set as the result. +// 1. Caller tries to get the code hash of a normal contract account, state +// should return the relative code hash and set it as the result. // -// (3) Caller tries to get the code hash for an account without contract code, -// state should return emptyCodeHash(0xc5d246...) as the result. +// 2. Caller tries to get the code hash of a non-existent account, state should +// return common.Hash{} and zero will be set as the result. // -// (4) Caller tries to get the code hash of a precompiled account, the result -// should be zero or emptyCodeHash. +// 3. Caller tries to get the code hash for an account without contract code, state +// should return emptyCodeHash(0xc5d246...) as the result. // -// It is worth noting that in order to avoid unnecessary create and clean, -// all precompile accounts on mainnet have been transferred 1 wei, so the return -// here should be emptyCodeHash. -// If the precompile account is not transferred any amount on a private or +// 4. Caller tries to get the code hash of a precompiled account, the result should be +// zero or emptyCodeHash. +// +// It is worth noting that in order to avoid unnecessary create and clean, all precompile +// accounts on mainnet have been transferred 1 wei, so the return here should be +// emptyCodeHash. If the precompile account is not transferred any amount on a private or // customized chain, the return value will be zero. // -// (5) Caller tries to get the code hash for an account which is marked as suicided -// in the current transaction, the code hash of this account should be returned. +// 5. Caller tries to get the code hash for an account which is marked as suicided +// in the current transaction, the code hash of this account should be returned. // -// (6) Caller tries to get the code hash for an account which is marked as deleted, -// this account should be regarded as a non-existent account and zero should be returned. +// 6. Caller tries to get the code hash for an account which is marked as deleted, this +// account should be regarded as a non-existent account and zero should be returned. func opExtCodeHash(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { slot := scope.Stack.peek() address := common.Address(slot.Bytes20()) @@ -697,7 +697,6 @@ func opCall(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byt } stack.push(&temp) if err == nil || err == ErrExecutionReverted { - ret = common.CopyBytes(ret) scope.Memory.Set(retOffset.Uint64(), retSize.Uint64(), ret) } scope.Contract.Gas += returnGas @@ -733,7 +732,6 @@ func opCallCode(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([ } stack.push(&temp) if err == nil || err == ErrExecutionReverted { - ret = common.CopyBytes(ret) scope.Memory.Set(retOffset.Uint64(), retSize.Uint64(), ret) } scope.Contract.Gas += returnGas @@ -762,7 +760,6 @@ func opDelegateCall(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext } stack.push(&temp) if err == nil || err == ErrExecutionReverted { - ret = common.CopyBytes(ret) scope.Memory.Set(retOffset.Uint64(), retSize.Uint64(), ret) } scope.Contract.Gas += returnGas @@ -791,7 +788,6 @@ func opStaticCall(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) } stack.push(&temp) if err == nil || err == ErrExecutionReverted { - ret = common.CopyBytes(ret) scope.Memory.Set(retOffset.Uint64(), retSize.Uint64(), ret) } scope.Contract.Gas += returnGas diff --git a/core/vm/instructions_test.go b/core/vm/instructions_test.go index f0fa4811ca7b5..602cde51015ea 100644 --- a/core/vm/instructions_test.go +++ b/core/vm/instructions_test.go @@ -46,7 +46,6 @@ var commonParams []*twoOperandParams var twoOpMethods map[string]executionFunc func init() { - // Params is a list of common edgecases that should be used for some common tests params := []string{ "0000000000000000000000000000000000000000000000000000000000000000", // 0 @@ -92,7 +91,6 @@ func init() { } func testTwoOperandOp(t *testing.T, tests []TwoOperandTestcase, opFn executionFunc, name string) { - var ( env = NewEVM(BlockContext{}, TxContext{}, nil, params.TestChainConfig, Config{}) stack = newstack() @@ -229,32 +227,32 @@ func TestAddMod(t *testing.T) { } } -// getResult is a convenience function to generate the expected values -func getResult(args []*twoOperandParams, opFn executionFunc) []TwoOperandTestcase { - var ( - env = NewEVM(BlockContext{}, TxContext{}, nil, params.TestChainConfig, Config{}) - stack = newstack() - pc = uint64(0) - interpreter = env.interpreter - ) - result := make([]TwoOperandTestcase, len(args)) - for i, param := range args { - x := new(uint256.Int).SetBytes(common.Hex2Bytes(param.x)) - y := new(uint256.Int).SetBytes(common.Hex2Bytes(param.y)) - stack.push(x) - stack.push(y) - opFn(&pc, interpreter, &ScopeContext{nil, stack, nil}) - actual := stack.pop() - result[i] = TwoOperandTestcase{param.x, param.y, fmt.Sprintf("%064x", actual)} - } - return result -} - // utility function to fill the json-file with testcases // Enable this test to generate the 'testcases_xx.json' files func TestWriteExpectedValues(t *testing.T) { t.Skip("Enable this test to create json test cases.") + // getResult is a convenience function to generate the expected values + getResult := func(args []*twoOperandParams, opFn executionFunc) []TwoOperandTestcase { + var ( + env = NewEVM(BlockContext{}, TxContext{}, nil, params.TestChainConfig, Config{}) + stack = newstack() + pc = uint64(0) + interpreter = env.interpreter + ) + result := make([]TwoOperandTestcase, len(args)) + for i, param := range args { + x := new(uint256.Int).SetBytes(common.Hex2Bytes(param.x)) + y := new(uint256.Int).SetBytes(common.Hex2Bytes(param.y)) + stack.push(x) + stack.push(y) + opFn(&pc, interpreter, &ScopeContext{nil, stack, nil}) + actual := stack.pop() + result[i] = TwoOperandTestcase{param.x, param.y, fmt.Sprintf("%064x", actual)} + } + return result + } + for name, method := range twoOpMethods { data, err := json.Marshal(getResult(commonParams, method)) if err != nil { @@ -641,7 +639,6 @@ func TestCreate2Addreses(t *testing.T) { expected: "0xE33C0C7F7df4809055C3ebA6c09CFe4BaF1BD9e0", }, } { - origin := common.BytesToAddress(common.FromHex(tt.origin)) salt := common.BytesToHash(common.FromHex(tt.salt)) code := common.FromHex(tt.code) diff --git a/core/vm/interpreter.go b/core/vm/interpreter.go index 4f1ebc43a229f..312977b755881 100644 --- a/core/vm/interpreter.go +++ b/core/vm/interpreter.go @@ -17,10 +17,9 @@ package vm import ( - "hash" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/math" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" ) @@ -44,21 +43,13 @@ type ScopeContext struct { Contract *Contract } -// keccakState wraps sha3.state. In addition to the usual hash methods, it also supports -// Read to get a variable amount of data from the hash state. Read is faster than Sum -// because it doesn't copy the internal state, but also modifies the internal state. -type keccakState interface { - hash.Hash - Read([]byte) (int, error) -} - // EVMInterpreter represents an EVM interpreter type EVMInterpreter struct { evm *EVM cfg Config - hasher keccakState // Keccak256 hasher instance shared across opcodes - hasherBuf common.Hash // Keccak256 hasher result array shared aross opcodes + hasher crypto.KeccakState // Keccak256 hasher instance shared across opcodes + hasherBuf common.Hash // Keccak256 hasher result array shared aross opcodes readOnly bool // Whether to throw on stateful modifications returnData []byte // Last CALL's return data for subsequent reuse @@ -90,15 +81,18 @@ func NewEVMInterpreter(evm *EVM, cfg Config) *EVMInterpreter { default: cfg.JumpTable = &frontierInstructionSet } - for i, eip := range cfg.ExtraEips { + var extraEips []int + for _, eip := range cfg.ExtraEips { copy := *cfg.JumpTable if err := EnableEIP(eip, ©); err != nil { // Disable it, so caller can check if it's activated or not - cfg.ExtraEips = append(cfg.ExtraEips[:i], cfg.ExtraEips[i+1:]...) log.Error("EIP activation failed", "eip", eip, "error", err) + } else { + extraEips = append(extraEips, eip) } cfg.JumpTable = © } + cfg.ExtraEips = extraEips } return &EVMInterpreter{ @@ -114,7 +108,6 @@ func NewEVMInterpreter(evm *EVM, cfg Config) *EVMInterpreter { // considered a revert-and-consume-all-gas operation except for // ErrExecutionReverted which means revert-and-keep-gas-left. func (in *EVMInterpreter) Run(contract *Contract, input []byte, readOnly bool) (ret []byte, err error) { - // Increment the call depth which is restricted to 1024 in.evm.depth++ defer func() { in.evm.depth-- }() diff --git a/core/vm/interpreter_test.go b/core/vm/interpreter_test.go index dfae0f2e2af7d..31ee9922dbac3 100644 --- a/core/vm/interpreter_test.go +++ b/core/vm/interpreter_test.go @@ -73,5 +73,4 @@ func TestLoopInterrupt(t *testing.T) { } } } - } diff --git a/core/vm/jump_table.go b/core/vm/jump_table.go index eef3b53d8c668..94229436d23c6 100644 --- a/core/vm/jump_table.go +++ b/core/vm/jump_table.go @@ -63,7 +63,7 @@ type JumpTable [256]*operation func validate(jt JumpTable) JumpTable { for i, op := range jt { if op == nil { - panic(fmt.Sprintf("op 0x%x is not set", i)) + panic(fmt.Sprintf("op %#x is not set", i)) } // The interpreter has an assumption that if the memorySize function is // set, then the dynamicGas function is also set. This is a somewhat @@ -80,7 +80,7 @@ func validate(jt JumpTable) JumpTable { func newMergeInstructionSet() JumpTable { instructionSet := newLondonInstructionSet() - instructionSet[RANDOM] = &operation{ + instructionSet[PREVRANDAO] = &operation{ execute: opRandom, constantGas: GasQuickStep, minStack: minStack(0, 1), @@ -198,7 +198,6 @@ func newSpuriousDragonInstructionSet() JumpTable { instructionSet := newTangerineWhistleInstructionSet() instructionSet[EXP].dynamicGas = gasExpEIP158 return validate(instructionSet) - } // EIP 150 a.k.a Tangerine Whistle diff --git a/core/vm/opcodes.go b/core/vm/opcodes.go index 19252b01f2569..fa7de5049ace1 100644 --- a/core/vm/opcodes.go +++ b/core/vm/opcodes.go @@ -25,11 +25,7 @@ type OpCode byte // IsPush specifies if an opcode is a PUSH opcode. func (op OpCode) IsPush() bool { - switch op { - case PUSH1, PUSH2, PUSH3, PUSH4, PUSH5, PUSH6, PUSH7, PUSH8, PUSH9, PUSH10, PUSH11, PUSH12, PUSH13, PUSH14, PUSH15, PUSH16, PUSH17, PUSH18, PUSH19, PUSH20, PUSH21, PUSH22, PUSH23, PUSH24, PUSH25, PUSH26, PUSH27, PUSH28, PUSH29, PUSH30, PUSH31, PUSH32: - return true - } - return false + return PUSH1 <= op && op <= PUSH32 } // 0x0 range - arithmetic ops. @@ -99,6 +95,7 @@ const ( NUMBER OpCode = 0x43 DIFFICULTY OpCode = 0x44 RANDOM OpCode = 0x44 // Same as DIFFICULTY + PREVRANDAO OpCode = 0x44 // Same as DIFFICULTY GASLIMIT OpCode = 0x45 CHAINID OpCode = 0x46 SELFBALANCE OpCode = 0x47 @@ -280,7 +277,7 @@ var opCodeToString = map[OpCode]string{ COINBASE: "COINBASE", TIMESTAMP: "TIMESTAMP", NUMBER: "NUMBER", - DIFFICULTY: "DIFFICULTY", // TODO (MariusVanDerWijden) rename to RANDOM post merge + DIFFICULTY: "DIFFICULTY", // TODO (MariusVanDerWijden) rename to PREVRANDAO post merge GASLIMIT: "GASLIMIT", CHAINID: "CHAINID", SELFBALANCE: "SELFBALANCE", @@ -392,7 +389,7 @@ var opCodeToString = map[OpCode]string{ func (op OpCode) String() string { str := opCodeToString[op] if len(str) == 0 { - return fmt.Sprintf("opcode 0x%x not defined", int(op)) + return fmt.Sprintf("opcode %#x not defined", int(op)) } return str diff --git a/core/vm/runtime/runtime_test.go b/core/vm/runtime/runtime_test.go index fcaa10f1c62cc..ab77e284df35f 100644 --- a/core/vm/runtime/runtime_test.go +++ b/core/vm/runtime/runtime_test.go @@ -22,7 +22,6 @@ import ( "os" "strings" "testing" - "time" "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" @@ -326,25 +325,6 @@ func TestBlockhash(t *testing.T) { } } -type stepCounter struct { - inner *logger.JSONLogger - steps int -} - -func (s *stepCounter) CaptureStart(env *vm.EVM, from common.Address, to common.Address, create bool, input []byte, gas uint64, value *big.Int) { -} - -func (s *stepCounter) CaptureFault(pc uint64, op vm.OpCode, gas, cost uint64, scope *vm.ScopeContext, depth int, err error) { -} - -func (s *stepCounter) CaptureEnd(output []byte, gasUsed uint64, t time.Duration, err error) {} - -func (s *stepCounter) CaptureState(pc uint64, op vm.OpCode, gas, cost uint64, scope *vm.ScopeContext, rData []byte, depth int, err error) { - s.steps++ - // Enable this for more output - //s.inner.CaptureState(env, pc, op, gas, cost, memory, stack, rStack, contract, depth, err) -} - // benchmarkNonModifyingCode benchmarks code, but if the code modifies the // state, this should not be used, since it does not reset the state between runs. func benchmarkNonModifyingCode(gas uint64, code []byte, name string, tracerCode string, b *testing.B) { @@ -353,7 +333,7 @@ func benchmarkNonModifyingCode(gas uint64, code []byte, name string, tracerCode cfg.State, _ = state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) cfg.GasLimit = gas if len(tracerCode) > 0 { - tracer, err := tracers.New(tracerCode, new(tracers.Context)) + tracer, err := tracers.New(tracerCode, new(tracers.Context), nil) if err != nil { b.Fatal(err) } @@ -399,7 +379,6 @@ func benchmarkNonModifyingCode(gas uint64, code []byte, name string, tracerCode // BenchmarkSimpleLoop test a pretty simple loop which loops until OOG // 55 ms func BenchmarkSimpleLoop(b *testing.B) { - staticCallIdentity := []byte{ byte(vm.JUMPDEST), // [ count ] // push args for the call @@ -478,7 +457,7 @@ func BenchmarkSimpleLoop(b *testing.B) { byte(vm.JUMP), } - calllRevertingContractWithInput := []byte{ + callRevertingContractWithInput := []byte{ byte(vm.JUMPDEST), // // push args for the call byte(vm.PUSH1), 0, // out size @@ -506,7 +485,7 @@ func BenchmarkSimpleLoop(b *testing.B) { benchmarkNonModifyingCode(100000000, loopingCode, "loop-100M", "", b) benchmarkNonModifyingCode(100000000, callInexistant, "call-nonexist-100M", "", b) benchmarkNonModifyingCode(100000000, callEOA, "call-EOA-100M", "", b) - benchmarkNonModifyingCode(100000000, calllRevertingContractWithInput, "call-reverting-100M", "", b) + benchmarkNonModifyingCode(100000000, callRevertingContractWithInput, "call-reverting-100M", "", b) //benchmarkNonModifyingCode(10000000, staticCallIdentity, "staticcall-identity-10M", b) //benchmarkNonModifyingCode(10000000, loopingCode, "loop-10M", b) @@ -518,12 +497,11 @@ func TestEip2929Cases(t *testing.T) { t.Skip("Test only useful for generating documentation") id := 1 prettyPrint := func(comment string, code []byte) { - instrs := make([]string, 0) it := asm.NewInstructionIterator(code) for it.Next() { if it.Arg() != nil && 0 < len(it.Arg()) { - instrs = append(instrs, fmt.Sprintf("%v 0x%x", it.Op(), it.Arg())) + instrs = append(instrs, fmt.Sprintf("%v %#x", it.Op(), it.Arg())) } else { instrs = append(instrs, fmt.Sprintf("%v", it.Op())) } @@ -531,7 +509,7 @@ func TestEip2929Cases(t *testing.T) { ops := strings.Join(instrs, ", ") fmt.Printf("### Case %d\n\n", id) id++ - fmt.Printf("%v\n\nBytecode: \n```\n0x%x\n```\nOperations: \n```\n%v\n```\n\n", + fmt.Printf("%v\n\nBytecode: \n```\n%#x\n```\nOperations: \n```\n%v\n```\n\n", comment, code, ops) Execute(code, nil, &Config{ @@ -854,7 +832,7 @@ func TestRuntimeJSTracer(t *testing.T) { statedb.SetCode(common.HexToAddress("0xee"), calleeCode) statedb.SetCode(common.HexToAddress("0xff"), depressedCode) - tracer, err := tracers.New(jsTracer, new(tracers.Context)) + tracer, err := tracers.New(jsTracer, new(tracers.Context), nil) if err != nil { t.Fatal(err) } @@ -890,7 +868,7 @@ func TestJSTracerCreateTx(t *testing.T) { code := []byte{byte(vm.PUSH1), 0, byte(vm.PUSH1), 0, byte(vm.RETURN)} statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) - tracer, err := tracers.New(jsTracer, new(tracers.Context)) + tracer, err := tracers.New(jsTracer, new(tracers.Context), nil) if err != nil { t.Fatal(err) } diff --git a/crypto/blake2b/blake2b.go b/crypto/blake2b/blake2b.go index 5da50cab6f00e..7ecaab813999f 100644 --- a/crypto/blake2b/blake2b.go +++ b/crypto/blake2b/blake2b.go @@ -302,6 +302,7 @@ func appendUint64(b []byte, x uint64) []byte { return append(b, a[:]...) } +//nolint:unused,deadcode func appendUint32(b []byte, x uint32) []byte { var a [4]byte binary.BigEndian.PutUint32(a[:], x) @@ -313,6 +314,7 @@ func consumeUint64(b []byte) ([]byte, uint64) { return b[8:], x } +//nolint:unused,deadcode func consumeUint32(b []byte) ([]byte, uint32) { x := binary.BigEndian.Uint32(b) return b[4:], x diff --git a/crypto/blake2b/blake2b_generic.go b/crypto/blake2b/blake2b_generic.go index 35c40cc924f8d..61e678fdf5763 100644 --- a/crypto/blake2b/blake2b_generic.go +++ b/crypto/blake2b/blake2b_generic.go @@ -25,6 +25,7 @@ var precomputed = [10][16]byte{ {10, 8, 7, 1, 2, 4, 6, 5, 15, 9, 3, 13, 11, 14, 12, 0}, } +// nolint:unused,deadcode func hashBlocksGeneric(h *[8]uint64, c *[2]uint64, flag uint64, blocks []byte) { var m [16]uint64 c0, c1 := c[0], c[1] diff --git a/crypto/blake2b/blake2b_test.go b/crypto/blake2b/blake2b_test.go index 9e7297da160f3..9d24444a27b7e 100644 --- a/crypto/blake2b/blake2b_test.go +++ b/crypto/blake2b/blake2b_test.go @@ -14,14 +14,6 @@ import ( "testing" ) -func fromHex(s string) []byte { - b, err := hex.DecodeString(s) - if err != nil { - panic(err) - } - return b -} - func TestHashes(t *testing.T) { defer func(sse4, avx, avx2 bool) { useSSE4, useAVX, useAVX2 = sse4, avx, avx2 diff --git a/crypto/bls12381/field_element_test.go b/crypto/bls12381/field_element_test.go index 0f6abd280cbb0..70bbe5cfe5e71 100644 --- a/crypto/bls12381/field_element_test.go +++ b/crypto/bls12381/field_element_test.go @@ -102,7 +102,6 @@ func TestFieldElementEquality(t *testing.T) { if a12.equal(b12) { t.Fatal("a != a + 1") } - } func TestFieldElementHelpers(t *testing.T) { diff --git a/crypto/bls12381/fp12.go b/crypto/bls12381/fp12.go index 3141c76c3995a..51e949fe5f049 100644 --- a/crypto/bls12381/fp12.go +++ b/crypto/bls12381/fp12.go @@ -96,7 +96,6 @@ func (e *fp12) add(c, a, b *fe12) { fp6 := e.fp6 fp6.add(&c[0], &a[0], &b[0]) fp6.add(&c[1], &a[1], &b[1]) - } func (e *fp12) double(c, a *fe12) { @@ -109,7 +108,6 @@ func (e *fp12) sub(c, a, b *fe12) { fp6 := e.fp6 fp6.sub(&c[0], &a[0], &b[0]) fp6.sub(&c[1], &a[1], &b[1]) - } func (e *fp12) neg(c, a *fe12) { diff --git a/crypto/bls12381/fp_test.go b/crypto/bls12381/fp_test.go index 97528d9db32ea..0bad35de16304 100644 --- a/crypto/bls12381/fp_test.go +++ b/crypto/bls12381/fp_test.go @@ -465,7 +465,6 @@ func TestFpNonResidue(t *testing.T) { i -= 1 } } - } func TestFp2Serialization(t *testing.T) { diff --git a/crypto/bls12381/g1.go b/crypto/bls12381/g1.go index d853823cd2987..52e12cc3a259d 100644 --- a/crypto/bls12381/g1.go +++ b/crypto/bls12381/g1.go @@ -228,7 +228,7 @@ func (g *G1) IsAffine(p *PointG1) bool { return p[2].isOne() } -// Add adds two G1 points p1, p2 and assigns the result to point at first argument. +// Affine calculates affine form of given G1 point. func (g *G1) Affine(p *PointG1) *PointG1 { if g.IsZero(p) { return p diff --git a/crypto/bls12381/g2.go b/crypto/bls12381/g2.go index fa110e3edfc5b..c2ca959bcca1b 100644 --- a/crypto/bls12381/g2.go +++ b/crypto/bls12381/g2.go @@ -41,7 +41,6 @@ func (p *PointG2) Zero() *PointG2 { p[1].one() p[2].zero() return p - } type tempG2 struct { diff --git a/crypto/bls12381/isogeny.go b/crypto/bls12381/isogeny.go index c3cb0a6f7bf07..a63f585dd00a0 100644 --- a/crypto/bls12381/isogeny.go +++ b/crypto/bls12381/isogeny.go @@ -19,7 +19,7 @@ package bls12381 // isogenyMapG1 applies 11-isogeny map for BLS12-381 G1 defined at draft-irtf-cfrg-hash-to-curve-06. func isogenyMapG1(x, y *fe) { // https://tools.ietf.org/html/draft-irtf-cfrg-hash-to-curve-06#appendix-C.2 - params := isogenyConstansG1 + params := isogenyConstantsG1 degree := 15 xNum, xDen, yNum, yDen := new(fe), new(fe), new(fe), new(fe) xNum.set(params[0][degree]) @@ -76,7 +76,7 @@ func isogenyMapG2(e *fp2, x, y *fe2) { y.set(yNum) } -var isogenyConstansG1 = [4][16]*fe{ +var isogenyConstantsG1 = [4][16]*fe{ { {0x4d18b6f3af00131c, 0x19fa219793fee28c, 0x3f2885f1467f19ae, 0x23dcea34f2ffb304, 0xd15b58d2ffc00054, 0x0913be200a20bef4}, {0x898985385cdbbd8b, 0x3c79e43cc7d966aa, 0x1597e193f4cd233a, 0x8637ef1e4d6623ad, 0x11b22deed20d827b, 0x07097bc5998784ad}, diff --git a/crypto/bn256/cloudflare/gfp_decl.go b/crypto/bn256/cloudflare/gfp_decl.go index ec4018e88a0c0..cf7f5654239fa 100644 --- a/crypto/bn256/cloudflare/gfp_decl.go +++ b/crypto/bn256/cloudflare/gfp_decl.go @@ -10,7 +10,7 @@ import ( "golang.org/x/sys/cpu" ) -//nolint:varcheck +//nolint:varcheck,unused,deadcode var hasBMI2 = cpu.X86.HasBMI2 // go:noescape diff --git a/crypto/crypto.go b/crypto/crypto.go index 45ea72747e6d9..e51b63becac96 100644 --- a/crypto/crypto.go +++ b/crypto/crypto.go @@ -35,7 +35,7 @@ import ( "golang.org/x/crypto/sha3" ) -//SignatureLength indicates the byte length required to carry a signature with recovery id. +// SignatureLength indicates the byte length required to carry a signature with recovery id. const SignatureLength = 64 + 1 // 64 bytes ECDSA signature + 1 byte recovery id // RecoveryIDOffset points to the byte offset within the signature that contains the recovery id. diff --git a/crypto/ecies/ecies_test.go b/crypto/ecies/ecies_test.go index 96e33da006fb8..8ca42c9c8ee63 100644 --- a/crypto/ecies/ecies_test.go +++ b/crypto/ecies/ecies_test.go @@ -334,7 +334,6 @@ func testParamSelection(t *testing.T, c testCase) { if err == nil { t.Fatalf("ecies: encryption should not have succeeded (%s)\n", c.Name) } - } // Ensure that the basic public key validation in the decryption operation diff --git a/crypto/secp256k1/curve.go b/crypto/secp256k1/curve.go index fa1b199a3484a..9b26ab292859e 100644 --- a/crypto/secp256k1/curve.go +++ b/crypto/secp256k1/curve.go @@ -105,7 +105,6 @@ func (BitCurve *BitCurve) IsOnCurve(x, y *big.Int) bool { return x3.Cmp(y2) == 0 } -//TODO: double check if the function is okay // affineFromJacobian reverses the Jacobian transform. See the comment at the // top of the file. func (BitCurve *BitCurve) affineFromJacobian(x, y, z *big.Int) (xOut, yOut *big.Int) { diff --git a/crypto/secp256k1/libsecp256k1/contrib/dummy.go b/crypto/secp256k1/libsecp256k1/contrib/dummy.go index fda594be99141..2c946210c54d3 100644 --- a/crypto/secp256k1/libsecp256k1/contrib/dummy.go +++ b/crypto/secp256k1/libsecp256k1/contrib/dummy.go @@ -1,3 +1,4 @@ +//go:build dummy // +build dummy // Package c contains only a C file. diff --git a/crypto/secp256k1/libsecp256k1/dummy.go b/crypto/secp256k1/libsecp256k1/dummy.go index 379b16992f474..04bbe3d76eccd 100644 --- a/crypto/secp256k1/libsecp256k1/dummy.go +++ b/crypto/secp256k1/libsecp256k1/dummy.go @@ -1,3 +1,4 @@ +//go:build dummy // +build dummy // Package c contains only a C file. diff --git a/crypto/secp256k1/libsecp256k1/include/dummy.go b/crypto/secp256k1/libsecp256k1/include/dummy.go index 5af540c73c4a5..64c71b8451d87 100644 --- a/crypto/secp256k1/libsecp256k1/include/dummy.go +++ b/crypto/secp256k1/libsecp256k1/include/dummy.go @@ -1,3 +1,4 @@ +//go:build dummy // +build dummy // Package c contains only a C file. diff --git a/crypto/secp256k1/libsecp256k1/src/dummy.go b/crypto/secp256k1/libsecp256k1/src/dummy.go index 65868f38a8ea9..2df270adc35e8 100644 --- a/crypto/secp256k1/libsecp256k1/src/dummy.go +++ b/crypto/secp256k1/libsecp256k1/src/dummy.go @@ -1,3 +1,4 @@ +//go:build dummy // +build dummy // Package c contains only a C file. diff --git a/crypto/secp256k1/libsecp256k1/src/modules/dummy.go b/crypto/secp256k1/libsecp256k1/src/modules/dummy.go index 3c7a696439f0e..99c538db51b04 100644 --- a/crypto/secp256k1/libsecp256k1/src/modules/dummy.go +++ b/crypto/secp256k1/libsecp256k1/src/modules/dummy.go @@ -1,3 +1,4 @@ +//go:build dummy // +build dummy // Package c contains only a C file. diff --git a/crypto/secp256k1/libsecp256k1/src/modules/ecdh/dummy.go b/crypto/secp256k1/libsecp256k1/src/modules/ecdh/dummy.go index b6fc38327ec8b..48c2e0aa54536 100644 --- a/crypto/secp256k1/libsecp256k1/src/modules/ecdh/dummy.go +++ b/crypto/secp256k1/libsecp256k1/src/modules/ecdh/dummy.go @@ -1,3 +1,4 @@ +//go:build dummy // +build dummy // Package c contains only a C file. diff --git a/crypto/secp256k1/libsecp256k1/src/modules/recovery/dummy.go b/crypto/secp256k1/libsecp256k1/src/modules/recovery/dummy.go index b9491f0cb9f48..8efbd7abe71bd 100644 --- a/crypto/secp256k1/libsecp256k1/src/modules/recovery/dummy.go +++ b/crypto/secp256k1/libsecp256k1/src/modules/recovery/dummy.go @@ -1,3 +1,4 @@ +//go:build dummy // +build dummy // Package c contains only a C file. diff --git a/eth/api.go b/eth/api.go index ef69acb76eb45..e480dde8f64f1 100644 --- a/eth/api.go +++ b/eth/api.go @@ -41,57 +41,44 @@ import ( "github.com/ethereum/go-ethereum/trie" ) -// PublicEthereumAPI provides an API to access Ethereum full node-related -// information. -type PublicEthereumAPI struct { +// EthereumAPI provides an API to access Ethereum full node-related information. +type EthereumAPI struct { e *Ethereum } -// NewPublicEthereumAPI creates a new Ethereum protocol API for full nodes. -func NewPublicEthereumAPI(e *Ethereum) *PublicEthereumAPI { - return &PublicEthereumAPI{e} +// NewEthereumAPI creates a new Ethereum protocol API for full nodes. +func NewEthereumAPI(e *Ethereum) *EthereumAPI { + return &EthereumAPI{e} } -// Etherbase is the address that mining rewards will be send to -func (api *PublicEthereumAPI) Etherbase() (common.Address, error) { +// Etherbase is the address that mining rewards will be send to. +func (api *EthereumAPI) Etherbase() (common.Address, error) { return api.e.Etherbase() } -// Coinbase is the address that mining rewards will be send to (alias for Etherbase) -func (api *PublicEthereumAPI) Coinbase() (common.Address, error) { +// Coinbase is the address that mining rewards will be send to (alias for Etherbase). +func (api *EthereumAPI) Coinbase() (common.Address, error) { return api.Etherbase() } -// Hashrate returns the POW hashrate -func (api *PublicEthereumAPI) Hashrate() hexutil.Uint64 { +// Hashrate returns the POW hashrate. +func (api *EthereumAPI) Hashrate() hexutil.Uint64 { return hexutil.Uint64(api.e.Miner().Hashrate()) } -// PublicMinerAPI provides an API to control the miner. -// It offers only methods that operate on data that pose no security risk when it is publicly accessible. -type PublicMinerAPI struct { - e *Ethereum -} - -// NewPublicMinerAPI create a new PublicMinerAPI instance. -func NewPublicMinerAPI(e *Ethereum) *PublicMinerAPI { - return &PublicMinerAPI{e} -} - // Mining returns an indication if this node is currently mining. -func (api *PublicMinerAPI) Mining() bool { +func (api *EthereumAPI) Mining() bool { return api.e.IsMining() } -// PrivateMinerAPI provides private RPC methods to control the miner. -// These methods can be abused by external users and must be considered insecure for use by untrusted users. -type PrivateMinerAPI struct { +// MinerAPI provides an API to control the miner. +type MinerAPI struct { e *Ethereum } -// NewPrivateMinerAPI create a new RPC service which controls the miner of this node. -func NewPrivateMinerAPI(e *Ethereum) *PrivateMinerAPI { - return &PrivateMinerAPI{e: e} +// NewMinerAPI create a new MinerAPI instance. +func NewMinerAPI(e *Ethereum) *MinerAPI { + return &MinerAPI{e} } // Start starts the miner with the given number of threads. If threads is nil, @@ -99,7 +86,7 @@ func NewPrivateMinerAPI(e *Ethereum) *PrivateMinerAPI { // usable by this process. If mining is already running, this method adjust the // number of threads allowed to use and updates the minimum price required by the // transaction pool. -func (api *PrivateMinerAPI) Start(threads *int) error { +func (api *MinerAPI) Start(threads *int) error { if threads == nil { return api.e.StartMining(runtime.NumCPU()) } @@ -108,12 +95,12 @@ func (api *PrivateMinerAPI) Start(threads *int) error { // Stop terminates the miner, both at the consensus engine level as well as at // the block creation level. -func (api *PrivateMinerAPI) Stop() { +func (api *MinerAPI) Stop() { api.e.StopMining() } // SetExtra sets the extra data string that is included when this miner mines a block. -func (api *PrivateMinerAPI) SetExtra(extra string) (bool, error) { +func (api *MinerAPI) SetExtra(extra string) (bool, error) { if err := api.e.Miner().SetExtra([]byte(extra)); err != nil { return false, err } @@ -121,7 +108,7 @@ func (api *PrivateMinerAPI) SetExtra(extra string) (bool, error) { } // SetGasPrice sets the minimum accepted gas price for the miner. -func (api *PrivateMinerAPI) SetGasPrice(gasPrice hexutil.Big) bool { +func (api *MinerAPI) SetGasPrice(gasPrice hexutil.Big) bool { api.e.lock.Lock() api.e.gasPrice = (*big.Int)(&gasPrice) api.e.lock.Unlock() @@ -131,37 +118,36 @@ func (api *PrivateMinerAPI) SetGasPrice(gasPrice hexutil.Big) bool { } // SetGasLimit sets the gaslimit to target towards during mining. -func (api *PrivateMinerAPI) SetGasLimit(gasLimit hexutil.Uint64) bool { +func (api *MinerAPI) SetGasLimit(gasLimit hexutil.Uint64) bool { api.e.Miner().SetGasCeil(uint64(gasLimit)) return true } -// SetEtherbase sets the etherbase of the miner -func (api *PrivateMinerAPI) SetEtherbase(etherbase common.Address) bool { +// SetEtherbase sets the etherbase of the miner. +func (api *MinerAPI) SetEtherbase(etherbase common.Address) bool { api.e.SetEtherbase(etherbase) return true } // SetRecommitInterval updates the interval for miner sealing work recommitting. -func (api *PrivateMinerAPI) SetRecommitInterval(interval int) { +func (api *MinerAPI) SetRecommitInterval(interval int) { api.e.Miner().SetRecommitInterval(time.Duration(interval) * time.Millisecond) } -// PrivateAdminAPI is the collection of Ethereum full node-related APIs -// exposed over the private admin endpoint. -type PrivateAdminAPI struct { +// AdminAPI is the collection of Ethereum full node related APIs for node +// administration. +type AdminAPI struct { eth *Ethereum } -// NewPrivateAdminAPI creates a new API definition for the full node private -// admin methods of the Ethereum service. -func NewPrivateAdminAPI(eth *Ethereum) *PrivateAdminAPI { - return &PrivateAdminAPI{eth: eth} +// NewAdminAPI creates a new instance of AdminAPI. +func NewAdminAPI(eth *Ethereum) *AdminAPI { + return &AdminAPI{eth: eth} } // ExportChain exports the current blockchain into a local file, -// or a range of blocks if first and last are non-nil -func (api *PrivateAdminAPI) ExportChain(file string, first *uint64, last *uint64) (bool, error) { +// or a range of blocks if first and last are non-nil. +func (api *AdminAPI) ExportChain(file string, first *uint64, last *uint64) (bool, error) { if first == nil && last != nil { return false, errors.New("last cannot be specified without first") } @@ -171,7 +157,7 @@ func (api *PrivateAdminAPI) ExportChain(file string, first *uint64, last *uint64 } if _, err := os.Stat(file); err == nil { // File already exists. Allowing overwrite could be a DoS vector, - // since the 'file' may point to arbitrary paths on the drive + // since the 'file' may point to arbitrary paths on the drive. return false, errors.New("location would overwrite an existing file") } // Make sure we can create the file to export into @@ -209,7 +195,7 @@ func hasAllBlocks(chain *core.BlockChain, bs []*types.Block) bool { } // ImportChain imports a blockchain from a local file. -func (api *PrivateAdminAPI) ImportChain(file string) (bool, error) { +func (api *AdminAPI) ImportChain(file string) (bool, error) { // Make sure the can access the file to import in, err := os.Open(file) if err != nil { @@ -257,20 +243,19 @@ func (api *PrivateAdminAPI) ImportChain(file string) (bool, error) { return true, nil } -// PublicDebugAPI is the collection of Ethereum full node APIs exposed -// over the public debugging endpoint. -type PublicDebugAPI struct { +// DebugAPI is the collection of Ethereum full node APIs for debugging the +// protocol. +type DebugAPI struct { eth *Ethereum } -// NewPublicDebugAPI creates a new API definition for the full node- -// related public debug methods of the Ethereum service. -func NewPublicDebugAPI(eth *Ethereum) *PublicDebugAPI { - return &PublicDebugAPI{eth: eth} +// NewDebugAPI creates a new DebugAPI instance. +func NewDebugAPI(eth *Ethereum) *DebugAPI { + return &DebugAPI{eth: eth} } // DumpBlock retrieves the entire state of the database at a given block. -func (api *PublicDebugAPI) DumpBlock(blockNr rpc.BlockNumber) (state.Dump, error) { +func (api *DebugAPI) DumpBlock(blockNr rpc.BlockNumber) (state.Dump, error) { opts := &state.DumpConfig{ OnlyWithAddresses: true, Max: AccountRangeMaxResults, // Sanity limit over RPC @@ -287,6 +272,8 @@ func (api *PublicDebugAPI) DumpBlock(blockNr rpc.BlockNumber) (state.Dump, error block = api.eth.blockchain.CurrentBlock() } else if blockNr == rpc.FinalizedBlockNumber { block = api.eth.blockchain.CurrentFinalizedBlock() + } else if blockNr == rpc.SafeBlockNumber { + block = api.eth.blockchain.CurrentSafeBlock() } else { block = api.eth.blockchain.GetBlockByNumber(uint64(blockNr)) } @@ -300,20 +287,8 @@ func (api *PublicDebugAPI) DumpBlock(blockNr rpc.BlockNumber) (state.Dump, error return stateDb.RawDump(opts), nil } -// PrivateDebugAPI is the collection of Ethereum full node APIs exposed over -// the private debugging endpoint. -type PrivateDebugAPI struct { - eth *Ethereum -} - -// NewPrivateDebugAPI creates a new API definition for the full node-related -// private debug methods of the Ethereum service. -func NewPrivateDebugAPI(eth *Ethereum) *PrivateDebugAPI { - return &PrivateDebugAPI{eth: eth} -} - // Preimage is a debug API function that returns the preimage for a sha3 hash, if known. -func (api *PrivateDebugAPI) Preimage(ctx context.Context, hash common.Hash) (hexutil.Bytes, error) { +func (api *DebugAPI) Preimage(ctx context.Context, hash common.Hash) (hexutil.Bytes, error) { if preimage := rawdb.ReadPreimage(api.eth.ChainDb(), hash); preimage != nil { return preimage, nil } @@ -328,8 +303,8 @@ type BadBlockArgs struct { } // GetBadBlocks returns a list of the last 'bad blocks' that the client has seen on the network -// and returns them as a JSON list of block-hashes -func (api *PrivateDebugAPI) GetBadBlocks(ctx context.Context) ([]*BadBlockArgs, error) { +// and returns them as a JSON list of block hashes. +func (api *DebugAPI) GetBadBlocks(ctx context.Context) ([]*BadBlockArgs, error) { var ( err error blocks = rawdb.ReadAllBadBlocks(api.eth.chainDb) @@ -343,7 +318,7 @@ func (api *PrivateDebugAPI) GetBadBlocks(ctx context.Context) ([]*BadBlockArgs, if rlpBytes, err := rlp.EncodeToBytes(block); err != nil { blockRlp = err.Error() // Hacky, but hey, it works } else { - blockRlp = fmt.Sprintf("0x%x", rlpBytes) + blockRlp = fmt.Sprintf("%#x", rlpBytes) } if blockJSON, err = ethapi.RPCMarshalBlock(block, true, true, api.eth.APIBackend.ChainConfig()); err != nil { blockJSON = map[string]interface{}{"error": err.Error()} @@ -361,7 +336,7 @@ func (api *PrivateDebugAPI) GetBadBlocks(ctx context.Context) ([]*BadBlockArgs, const AccountRangeMaxResults = 256 // AccountRange enumerates all accounts in the given block and start point in paging request -func (api *PublicDebugAPI) AccountRange(blockNrOrHash rpc.BlockNumberOrHash, start []byte, maxResults int, nocode, nostorage, incompletes bool) (state.IteratorDump, error) { +func (api *DebugAPI) AccountRange(blockNrOrHash rpc.BlockNumberOrHash, start hexutil.Bytes, maxResults int, nocode, nostorage, incompletes bool) (state.IteratorDump, error) { var stateDb *state.StateDB var err error @@ -377,6 +352,8 @@ func (api *PublicDebugAPI) AccountRange(blockNrOrHash rpc.BlockNumberOrHash, sta block = api.eth.blockchain.CurrentBlock() } else if number == rpc.FinalizedBlockNumber { block = api.eth.blockchain.CurrentFinalizedBlock() + } else if number == rpc.SafeBlockNumber { + block = api.eth.blockchain.CurrentSafeBlock() } else { block = api.eth.blockchain.GetBlockByNumber(uint64(number)) } @@ -428,16 +405,18 @@ type storageEntry struct { } // StorageRangeAt returns the storage at the given block height and transaction index. -func (api *PrivateDebugAPI) StorageRangeAt(blockHash common.Hash, txIndex int, contractAddress common.Address, keyStart hexutil.Bytes, maxResult int) (StorageRangeResult, error) { +func (api *DebugAPI) StorageRangeAt(blockHash common.Hash, txIndex int, contractAddress common.Address, keyStart hexutil.Bytes, maxResult int) (StorageRangeResult, error) { // Retrieve the block block := api.eth.blockchain.GetBlockByHash(blockHash) if block == nil { return StorageRangeResult{}, fmt.Errorf("block %#x not found", blockHash) } - _, _, statedb, err := api.eth.stateAtTransaction(block, txIndex, 0) + _, _, statedb, release, err := api.eth.stateAtTransaction(block, txIndex, 0) if err != nil { return StorageRangeResult{}, err } + defer release() + st := statedb.StorageTrie(contractAddress) if st == nil { return StorageRangeResult{}, fmt.Errorf("account %x doesn't exist", contractAddress) @@ -473,7 +452,7 @@ func storageRangeAt(st state.Trie, start []byte, maxResult int) (StorageRangeRes // code hash, or storage hash. // // With one parameter, returns the list of accounts modified in the specified block. -func (api *PrivateDebugAPI) GetModifiedAccountsByNumber(startNum uint64, endNum *uint64) ([]common.Address, error) { +func (api *DebugAPI) GetModifiedAccountsByNumber(startNum uint64, endNum *uint64) ([]common.Address, error) { var startBlock, endBlock *types.Block startBlock = api.eth.blockchain.GetBlockByNumber(startNum) @@ -501,7 +480,7 @@ func (api *PrivateDebugAPI) GetModifiedAccountsByNumber(startNum uint64, endNum // code hash, or storage hash. // // With one parameter, returns the list of accounts modified in the specified block. -func (api *PrivateDebugAPI) GetModifiedAccountsByHash(startHash common.Hash, endHash *common.Hash) ([]common.Address, error) { +func (api *DebugAPI) GetModifiedAccountsByHash(startHash common.Hash, endHash *common.Hash) ([]common.Address, error) { var startBlock, endBlock *types.Block startBlock = api.eth.blockchain.GetBlockByHash(startHash) if startBlock == nil { @@ -523,17 +502,17 @@ func (api *PrivateDebugAPI) GetModifiedAccountsByHash(startHash common.Hash, end return api.getModifiedAccounts(startBlock, endBlock) } -func (api *PrivateDebugAPI) getModifiedAccounts(startBlock, endBlock *types.Block) ([]common.Address, error) { +func (api *DebugAPI) getModifiedAccounts(startBlock, endBlock *types.Block) ([]common.Address, error) { if startBlock.Number().Uint64() >= endBlock.Number().Uint64() { return nil, fmt.Errorf("start block height (%d) must be less than end block height (%d)", startBlock.Number().Uint64(), endBlock.Number().Uint64()) } triedb := api.eth.BlockChain().StateCache().TrieDB() - oldTrie, err := trie.NewSecure(startBlock.Root(), triedb) + oldTrie, err := trie.NewStateTrie(trie.StateTrieID(startBlock.Root()), triedb) if err != nil { return nil, err } - newTrie, err := trie.NewSecure(endBlock.Root(), triedb) + newTrie, err := trie.NewStateTrie(trie.StateTrieID(endBlock.Root()), triedb) if err != nil { return nil, err } @@ -556,7 +535,7 @@ func (api *PrivateDebugAPI) getModifiedAccounts(startBlock, endBlock *types.Bloc // of the next block. // The (from, to) parameters are the sequence of blocks to search, which can go // either forwards or backwards -func (api *PrivateDebugAPI) GetAccessibleState(from, to rpc.BlockNumber) (uint64, error) { +func (api *DebugAPI) GetAccessibleState(from, to rpc.BlockNumber) (uint64, error) { db := api.eth.ChainDb() var pivot uint64 if p := rawdb.ReadLastPivotNumber(db); p != nil { @@ -609,5 +588,5 @@ func (api *PrivateDebugAPI) GetAccessibleState(from, to rpc.BlockNumber) (uint64 return uint64(i), nil } } - return 0, fmt.Errorf("No state found") + return 0, errors.New("no state found") } diff --git a/eth/api_backend.go b/eth/api_backend.go index f942710e2d8d6..ccc0966f00a5d 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -19,7 +19,6 @@ package eth import ( "context" "errors" - "fmt" "math/big" "time" @@ -34,6 +33,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/eth/gasprice" + "github.com/ethereum/go-ethereum/eth/tracers" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/miner" @@ -74,7 +74,18 @@ func (b *EthAPIBackend) HeaderByNumber(ctx context.Context, number rpc.BlockNumb return b.eth.blockchain.CurrentBlock().Header(), nil } if number == rpc.FinalizedBlockNumber { - return b.eth.blockchain.CurrentFinalizedBlock().Header(), nil + block := b.eth.blockchain.CurrentFinalizedBlock() + if block != nil { + return block.Header(), nil + } + return nil, errors.New("finalized block not found") + } + if number == rpc.SafeBlockNumber { + block := b.eth.blockchain.CurrentSafeBlock() + if block != nil { + return block.Header(), nil + } + return nil, errors.New("safe block not found") } return b.eth.blockchain.GetHeaderByNumber(uint64(number)), nil } @@ -113,6 +124,9 @@ func (b *EthAPIBackend) BlockByNumber(ctx context.Context, number rpc.BlockNumbe if number == rpc.FinalizedBlockNumber { return b.eth.blockchain.CurrentFinalizedBlock(), nil } + if number == rpc.SafeBlockNumber { + return b.eth.blockchain.CurrentSafeBlock(), nil + } return b.eth.blockchain.GetBlockByNumber(uint64(number)), nil } @@ -188,17 +202,8 @@ func (b *EthAPIBackend) GetReceipts(ctx context.Context, hash common.Hash) (type return b.eth.blockchain.GetReceiptsByHash(hash), nil } -func (b *EthAPIBackend) GetLogs(ctx context.Context, hash common.Hash) ([][]*types.Log, error) { - db := b.eth.ChainDb() - number := rawdb.ReadHeaderNumber(db, hash) - if number == nil { - return nil, fmt.Errorf("failed to get block number for hash %#x", hash) - } - logs := rawdb.ReadLogs(db, hash, *number, b.eth.blockchain.Config()) - if logs == nil { - return nil, fmt.Errorf("failed to get logs for block #%d (0x%s)", *number, hash.TerminalString()) - } - return logs, nil +func (b *EthAPIBackend) GetLogs(ctx context.Context, hash common.Hash, number uint64) ([][]*types.Log, error) { + return rawdb.ReadLogs(b.eth.chainDb, hash, number, b.ChainConfig()), nil } func (b *EthAPIBackend) GetTd(ctx context.Context, hash common.Hash) *big.Int { @@ -209,13 +214,12 @@ func (b *EthAPIBackend) GetTd(ctx context.Context, hash common.Hash) *big.Int { } func (b *EthAPIBackend) GetEVM(ctx context.Context, msg core.Message, state *state.StateDB, header *types.Header, vmConfig *vm.Config) (*vm.EVM, func() error, error) { - vmError := func() error { return nil } if vmConfig == nil { vmConfig = b.eth.blockchain.GetVMConfig() } txContext := core.NewEVMTxContext(msg) context := core.NewEVMBlockContext(header, b.eth.BlockChain(), nil) - return vm.NewEVM(context, txContext, state, b.eth.blockchain.Config(), *vmConfig), vmError, nil + return vm.NewEVM(context, txContext, state, b.eth.blockchain.Config(), *vmConfig), state.Error, nil } func (b *EthAPIBackend) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription { @@ -359,10 +363,10 @@ func (b *EthAPIBackend) StartMining(threads int) error { return b.eth.StartMining(threads) } -func (b *EthAPIBackend) StateAtBlock(ctx context.Context, block *types.Block, reexec uint64, base *state.StateDB, checkLive, preferDisk bool) (*state.StateDB, error) { - return b.eth.StateAtBlock(block, reexec, base, checkLive, preferDisk) +func (b *EthAPIBackend) StateAtBlock(ctx context.Context, block *types.Block, reexec uint64, base *state.StateDB, readOnly bool, preferDisk bool) (*state.StateDB, tracers.StateReleaseFunc, error) { + return b.eth.StateAtBlock(block, reexec, base, readOnly, preferDisk) } -func (b *EthAPIBackend) StateAtTransaction(ctx context.Context, block *types.Block, txIndex int, reexec uint64) (core.Message, vm.BlockContext, *state.StateDB, error) { +func (b *EthAPIBackend) StateAtTransaction(ctx context.Context, block *types.Block, txIndex int, reexec uint64) (core.Message, vm.BlockContext, *state.StateDB, tracers.StateReleaseFunc, error) { return b.eth.stateAtTransaction(block, txIndex, reexec) } diff --git a/eth/api_test.go b/eth/api_test.go index 39a1d5846004c..250591c1079b6 100644 --- a/eth/api_test.go +++ b/eth/api_test.go @@ -29,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/trie" ) var dumper = spew.ConfigState{Indent: " "} @@ -66,7 +67,7 @@ func TestAccountRange(t *testing.T) { t.Parallel() var ( - statedb = state.NewDatabaseWithConfig(rawdb.NewMemoryDatabase(), nil) + statedb = state.NewDatabaseWithConfig(rawdb.NewMemoryDatabase(), &trie.Config{Preimages: true}) state, _ = state.New(common.Hash{}, statedb, nil) addrs = [AccountRangeMaxResults * 2]common.Address{} m = map[common.Address]bool{} @@ -213,7 +214,7 @@ func TestStorageRangeAt(t *testing.T) { t.Error(err) } if !reflect.DeepEqual(result, test.want) { - t.Fatalf("wrong result for range 0x%x.., limit %d:\ngot %s\nwant %s", + t.Fatalf("wrong result for range %#x.., limit %d:\ngot %s\nwant %s", test.start, test.limit, dumper.Sdump(result), dumper.Sdump(&test.want)) } } diff --git a/eth/backend.go b/eth/backend.go index 8e70723b5b160..dca96e4f9d6c4 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -24,7 +24,6 @@ import ( "runtime" "sync" "sync/atomic" - "time" "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/common" @@ -40,7 +39,6 @@ import ( "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/ethconfig" - "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/eth/gasprice" "github.com/ethereum/go-ethereum/eth/protocols/eth" "github.com/ethereum/go-ethereum/eth/protocols/snap" @@ -93,7 +91,7 @@ type Ethereum struct { etherbase common.Address networkID uint64 - netRPCService *ethapi.PublicNetAPI + netRPCService *ethapi.NetAPI p2pServer *p2p.Server @@ -127,32 +125,30 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { } log.Info("Allocated trie memory caches", "clean", common.StorageSize(config.TrieCleanCache)*1024*1024, "dirty", common.StorageSize(config.TrieDirtyCache)*1024*1024) - // Transfer mining-related config to the ethash config. - ethashConfig := config.Ethash - ethashConfig.NotifyFull = config.Miner.NotifyFull - // Assemble the Ethereum object chainDb, err := stack.OpenDatabaseWithFreezer("chaindata", config.DatabaseCache, config.DatabaseHandles, config.DatabaseFreezer, "eth/db/chaindata/", false) if err != nil { return nil, err } - chainConfig, genesisHash, genesisErr := core.SetupGenesisBlockWithOverride(chainDb, config.Genesis, config.OverrideArrowGlacier, config.OverrideTerminalTotalDifficulty) - if _, ok := genesisErr.(*params.ConfigCompatError); genesisErr != nil && !ok { - return nil, genesisErr - } - log.Info("Initialised chain configuration", "config", chainConfig) - if err := pruner.RecoverPruning(stack.ResolvePath(""), chainDb, stack.ResolvePath(config.TrieCleanCacheJournal)); err != nil { log.Error("Failed to recover state", "error", err) } - merger := consensus.NewMerger(chainDb) + // Transfer mining-related config to the ethash config. + ethashConfig := config.Ethash + ethashConfig.NotifyFull = config.Miner.NotifyFull + cliqueConfig, err := core.LoadCliqueConfig(chainDb, config.Genesis) + if err != nil { + return nil, err + } + engine := ethconfig.CreateConsensusEngine(stack, ðashConfig, cliqueConfig, config.Miner.Notify, config.Miner.Noverify, chainDb) + eth := &Ethereum{ config: config, - merger: merger, + merger: consensus.NewMerger(chainDb), chainDb: chainDb, eventMux: stack.EventMux(), accountManager: stack.AccountManager(), - engine: ethconfig.CreateConsensusEngine(stack, chainConfig, ðashConfig, config.Miner.Notify, config.Miner.Noverify, chainDb), + engine: engine, closeBloomHandler: make(chan struct{}), networkID: config.NetworkId, gasPrice: config.Miner.GasPrice, @@ -196,34 +192,36 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { Preimages: config.Preimages, } ) - eth.blockchain, err = core.NewBlockChain(chainDb, cacheConfig, chainConfig, eth.engine, vmConfig, eth.shouldPreserve, &config.TxLookupLimit) + // Override the chain config with provided settings. + var overrides core.ChainOverrides + if config.OverrideTerminalTotalDifficulty != nil { + overrides.OverrideTerminalTotalDifficulty = config.OverrideTerminalTotalDifficulty + } + if config.OverrideTerminalTotalDifficultyPassed != nil { + overrides.OverrideTerminalTotalDifficultyPassed = config.OverrideTerminalTotalDifficultyPassed + } + eth.blockchain, err = core.NewBlockChain(chainDb, cacheConfig, config.Genesis, &overrides, eth.engine, vmConfig, eth.shouldPreserve, &config.TxLookupLimit) if err != nil { return nil, err } - // Rewind the chain in case of an incompatible config upgrade. - if compat, ok := genesisErr.(*params.ConfigCompatError); ok { - log.Warn("Rewinding chain to upgrade configuration", "err", compat) - eth.blockchain.SetHead(compat.RewindTo) - rawdb.WriteChainConfig(chainDb, genesisHash, chainConfig) - } eth.bloomIndexer.Start(eth.blockchain) if config.TxPool.Journal != "" { config.TxPool.Journal = stack.ResolvePath(config.TxPool.Journal) } - eth.txPool = core.NewTxPool(config.TxPool, chainConfig, eth.blockchain) + eth.txPool = core.NewTxPool(config.TxPool, eth.blockchain.Config(), eth.blockchain) // Permit the downloader to use the trie cache allowance during fast sync cacheLimit := cacheConfig.TrieCleanLimit + cacheConfig.TrieDirtyLimit + cacheConfig.SnapshotLimit checkpoint := config.Checkpoint if checkpoint == nil { - checkpoint = params.TrustedCheckpoints[genesisHash] + checkpoint = params.TrustedCheckpoints[eth.blockchain.Genesis().Hash()] } if eth.handler, err = newHandler(&handlerConfig{ Database: chainDb, Chain: eth.blockchain, TxPool: eth.txPool, - Merger: merger, + Merger: eth.merger, Network: config.NetworkId, Sync: config.SyncMode, BloomCache: uint64(cacheLimit), @@ -234,7 +232,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { return nil, err } - eth.miner = miner.New(eth, &config.Miner, chainConfig, eth.EventMux(), eth.engine, eth.isLocalBlock) + eth.miner = miner.New(eth, &config.Miner, eth.blockchain.Config(), eth.EventMux(), eth.engine, eth.isLocalBlock) eth.miner.SetExtra(makeExtraData(config.Miner.ExtraData)) eth.APIBackend = &EthAPIBackend{stack.Config().ExtRPCEnabled(), stack.Config().AllowUnprotectedTxs, eth, nil} @@ -259,7 +257,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { } // Start the RPC service - eth.netRPCService = ethapi.NewPublicNetAPI(eth.p2pServer, config.NetworkId) + eth.netRPCService = ethapi.NewNetAPI(eth.p2pServer, config.NetworkId) // Register the backend on the node stack.RegisterAPIs(eth.APIs()) @@ -301,47 +299,22 @@ func (s *Ethereum) APIs() []rpc.API { return append(apis, []rpc.API{ { Namespace: "eth", - Version: "1.0", - Service: NewPublicEthereumAPI(s), - Public: true, - }, { - Namespace: "eth", - Version: "1.0", - Service: NewPublicMinerAPI(s), - Public: true, - }, { - Namespace: "eth", - Version: "1.0", - Service: downloader.NewPublicDownloaderAPI(s.handler.downloader, s.eventMux), - Public: true, + Service: NewEthereumAPI(s), }, { Namespace: "miner", - Version: "1.0", - Service: NewPrivateMinerAPI(s), - Public: false, + Service: NewMinerAPI(s), }, { Namespace: "eth", - Version: "1.0", - Service: filters.NewPublicFilterAPI(s.APIBackend, false, 5*time.Minute), - Public: true, + Service: downloader.NewDownloaderAPI(s.handler.downloader, s.eventMux), }, { Namespace: "admin", - Version: "1.0", - Service: NewPrivateAdminAPI(s), - }, { - Namespace: "debug", - Version: "1.0", - Service: NewPublicDebugAPI(s), - Public: true, + Service: NewAdminAPI(s), }, { Namespace: "debug", - Version: "1.0", - Service: NewPrivateDebugAPI(s), + Service: NewDebugAPI(s), }, { Namespace: "net", - Version: "1.0", Service: s.netRPCService, - Public: true, }, }...) } diff --git a/eth/catalyst/api.go b/eth/catalyst/api.go index 4ed5093d831c5..2756a02e2f6d9 100644 --- a/eth/catalyst/api.go +++ b/eth/catalyst/api.go @@ -22,6 +22,7 @@ import ( "encoding/binary" "errors" "fmt" + "math/big" "sync" "time" @@ -31,79 +32,153 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth" + "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/rpc" ) -// Register adds catalyst APIs to the full node. +// Register adds the engine API to the full node. func Register(stack *node.Node, backend *eth.Ethereum) error { - log.Warn("Catalyst mode enabled", "protocol", "eth") + log.Warn("Engine API enabled", "protocol", "eth") stack.RegisterAPIs([]rpc.API{ { Namespace: "engine", - Version: "1.0", Service: NewConsensusAPI(backend), - Public: true, Authenticated: true, }, - { - Namespace: "engine", - Version: "1.0", - Service: NewConsensusAPI(backend), - Public: true, - Authenticated: false, - }, }) return nil } +const ( + // invalidBlockHitEviction is the number of times an invalid block can be + // referenced in forkchoice update or new payload before it is attempted + // to be reprocessed again. + invalidBlockHitEviction = 128 + + // invalidTipsetsCap is the max number of recent block hashes tracked that + // have lead to some bad ancestor block. It's just an OOM protection. + invalidTipsetsCap = 512 + + // beaconUpdateStartupTimeout is the time to wait for a beacon client to get + // attached before starting to issue warnings. + beaconUpdateStartupTimeout = 30 * time.Second + + // beaconUpdateExchangeTimeout is the max time allowed for a beacon client to + // do a transition config exchange before it's considered offline and the user + // is warned. + beaconUpdateExchangeTimeout = 2 * time.Minute + + // beaconUpdateConsensusTimeout is the max time allowed for a beacon client + // to send a consensus update before it's considered offline and the user is + // warned. + beaconUpdateConsensusTimeout = 30 * time.Second + + // beaconUpdateWarnFrequency is the frequency at which to warn the user that + // the beacon client is offline. + beaconUpdateWarnFrequency = 5 * time.Minute +) + type ConsensusAPI struct { - eth *eth.Ethereum + eth *eth.Ethereum + remoteBlocks *headerQueue // Cache of remote payloads received localBlocks *payloadQueue // Cache of local payloads generated - // Lock for the forkChoiceUpdated method - forkChoiceLock sync.Mutex + + // The forkchoice update and new payload method require us to return the + // latest valid hash in an invalid chain. To support that return, we need + // to track historical bad blocks as well as bad tipsets in case a chain + // is constantly built on it. + // + // There are a few important caveats in this mechanism: + // - The bad block tracking is ephemeral, in-memory only. We must never + // persist any bad block information to disk as a bug in Geth could end + // up blocking a valid chain, even if a later Geth update would accept + // it. + // - Bad blocks will get forgotten after a certain threshold of import + // attempts and will be retried. The rationale is that if the network + // really-really-really tries to feed us a block, we should give it a + // new chance, perhaps us being racey instead of the block being legit + // bad (this happened in Geth at a point with import vs. pending race). + // - Tracking all the blocks built on top of the bad one could be a bit + // problematic, so we will only track the head chain segment of a bad + // chain to allow discarding progressing bad chains and side chains, + // without tracking too much bad data. + invalidBlocksHits map[common.Hash]int // Emhemeral cache to track invalid blocks and their hit count + invalidTipsets map[common.Hash]*types.Header // Ephemeral cache to track invalid tipsets and their bad ancestor + invalidLock sync.Mutex // Protects the invalid maps from concurrent access + + // Geth can appear to be stuck or do strange things if the beacon client is + // offline or is sending us strange data. Stash some update stats away so + // that we can warn the user and not have them open issues on our tracker. + lastTransitionUpdate time.Time + lastTransitionLock sync.Mutex + lastForkchoiceUpdate time.Time + lastForkchoiceLock sync.Mutex + lastNewPayloadUpdate time.Time + lastNewPayloadLock sync.Mutex + + forkchoiceLock sync.Mutex // Lock for the forkChoiceUpdated method + newPayloadLock sync.Mutex // Lock for the NewPayload method } // NewConsensusAPI creates a new consensus api for the given backend. // The underlying blockchain needs to have a valid terminal total difficulty set. func NewConsensusAPI(eth *eth.Ethereum) *ConsensusAPI { if eth.BlockChain().Config().TerminalTotalDifficulty == nil { - panic("Catalyst started without valid total difficulty") + log.Warn("Engine API started but chain not configured for merge yet") } - return &ConsensusAPI{ - eth: eth, - remoteBlocks: newHeaderQueue(), - localBlocks: newPayloadQueue(), + api := &ConsensusAPI{ + eth: eth, + remoteBlocks: newHeaderQueue(), + localBlocks: newPayloadQueue(), + invalidBlocksHits: make(map[common.Hash]int), + invalidTipsets: make(map[common.Hash]*types.Header), } + eth.Downloader().SetBadBlockCallback(api.setInvalidAncestor) + go api.heartbeat() + + return api } // ForkchoiceUpdatedV1 has several responsibilities: -// If the method is called with an empty head block: -// we return success, which can be used to check if the catalyst mode is enabled -// If the total difficulty was not reached: -// we return INVALID -// If the finalizedBlockHash is set: -// we check if we have the finalizedBlockHash in our db, if not we start a sync -// We try to set our blockchain to the headBlock -// If there are payloadAttributes: -// we try to assemble a block with the payloadAttributes and return its payloadID +// +// We try to set our blockchain to the headBlock. +// +// If the method is called with an empty head block: we return success, which can be used +// to check if the engine API is enabled. +// +// If the total difficulty was not reached: we return INVALID. +// +// If the finalizedBlockHash is set: we check if we have the finalizedBlockHash in our db, +// if not we start a sync. +// +// If there are payloadAttributes: we try to assemble a block with the payloadAttributes +// and return its payloadID. func (api *ConsensusAPI) ForkchoiceUpdatedV1(update beacon.ForkchoiceStateV1, payloadAttributes *beacon.PayloadAttributesV1) (beacon.ForkChoiceResponse, error) { - api.forkChoiceLock.Lock() - defer api.forkChoiceLock.Unlock() + api.forkchoiceLock.Lock() + defer api.forkchoiceLock.Unlock() log.Trace("Engine API request received", "method", "ForkchoiceUpdated", "head", update.HeadBlockHash, "finalized", update.FinalizedBlockHash, "safe", update.SafeBlockHash) if update.HeadBlockHash == (common.Hash{}) { log.Warn("Forkchoice requested update to zero hash") return beacon.STATUS_INVALID, nil // TODO(karalabe): Why does someone send us this? } + // Stash away the last update to warn the user if the beacon client goes offline + api.lastForkchoiceLock.Lock() + api.lastForkchoiceUpdate = time.Now() + api.lastForkchoiceLock.Unlock() // Check whether we have the block yet in our database or not. If not, we'll // need to either trigger a sync, or to reject this forkchoice update for a // reason. block := api.eth.BlockChain().GetBlockByHash(update.HeadBlockHash) if block == nil { + // If this block was previously invalidated, keep rejecting it here too + if res := api.checkInvalidAncestor(update.HeadBlockHash, update.HeadBlockHash); res != nil { + return beacon.ForkChoiceResponse{PayloadStatus: *res, PayloadID: nil}, nil + } // If the head hash is unknown (was not given to us in a newPayload request), // we cannot resolve the header, so not much to do. This could be extended in // the future to resolve from the `eth` network, but it's an unexpected case @@ -147,16 +222,26 @@ func (api *ConsensusAPI) ForkchoiceUpdatedV1(update beacon.ForkchoiceStateV1, pa return beacon.ForkChoiceResponse{PayloadStatus: beacon.INVALID_TERMINAL_BLOCK, PayloadID: nil}, nil } } - + valid := func(id *beacon.PayloadID) beacon.ForkChoiceResponse { + return beacon.ForkChoiceResponse{ + PayloadStatus: beacon.PayloadStatusV1{Status: beacon.VALID, LatestValidHash: &update.HeadBlockHash}, + PayloadID: id, + } + } if rawdb.ReadCanonicalHash(api.eth.ChainDb(), block.NumberU64()) != update.HeadBlockHash { // Block is not canonical, set head. if latestValid, err := api.eth.BlockChain().SetCanonical(block); err != nil { return beacon.ForkChoiceResponse{PayloadStatus: beacon.PayloadStatusV1{Status: beacon.INVALID, LatestValidHash: &latestValid}}, err } + } else if api.eth.BlockChain().CurrentBlock().Hash() == update.HeadBlockHash { + // If the specified head matches with our local head, do nothing and keep + // generating the payload. It's a special corner case that a few slots are + // missing and we are requested to generate the payload in slot. } else { // If the head block is already in our canonical chain, the beacon client is // probably resyncing. Ignore the update. log.Info("Ignoring beacon update to old head", "number", block.NumberU64(), "hash", update.HeadBlockHash, "age", common.PrettyAge(time.Unix(int64(block.Time()), 0)), "have", api.eth.BlockChain().CurrentBlock().NumberU64()) + return valid(nil), nil } api.eth.SetSynced() @@ -189,12 +274,8 @@ func (api *ConsensusAPI) ForkchoiceUpdatedV1(update beacon.ForkchoiceStateV1, pa log.Warn("Safe block not in canonical chain") return beacon.STATUS_INVALID, beacon.InvalidForkChoiceState.With(errors.New("safe block not in canonical chain")) } - } - valid := func(id *beacon.PayloadID) beacon.ForkChoiceResponse { - return beacon.ForkChoiceResponse{ - PayloadStatus: beacon.PayloadStatusV1{Status: beacon.VALID, LatestValidHash: &update.HeadBlockHash}, - PayloadID: id, - } + // Set the safe block + api.eth.BlockChain().SetSafe(safeBlock) } // If payload generation was requested, create a new block to be potentially // sealed by the beacon client. The payload will be requested later, and we @@ -223,15 +304,20 @@ func (api *ConsensusAPI) ForkchoiceUpdatedV1(update beacon.ForkchoiceStateV1, pa // ExchangeTransitionConfigurationV1 checks the given configuration against // the configuration of the node. func (api *ConsensusAPI) ExchangeTransitionConfigurationV1(config beacon.TransitionConfigurationV1) (*beacon.TransitionConfigurationV1, error) { + log.Trace("Engine API request received", "method", "ExchangeTransitionConfiguration", "ttd", config.TerminalTotalDifficulty) if config.TerminalTotalDifficulty == nil { return nil, errors.New("invalid terminal total difficulty") } + // Stash away the last update to warn the user if the beacon client goes offline + api.lastTransitionLock.Lock() + api.lastTransitionUpdate = time.Now() + api.lastTransitionLock.Unlock() + ttd := api.eth.BlockChain().Config().TerminalTotalDifficulty - if ttd.Cmp(config.TerminalTotalDifficulty.ToInt()) != 0 { + if ttd == nil || ttd.Cmp(config.TerminalTotalDifficulty.ToInt()) != 0 { log.Warn("Invalid TTD configured", "geth", ttd, "beacon", config.TerminalTotalDifficulty) return nil, fmt.Errorf("invalid ttd: execution %v consensus %v", ttd, config.TerminalTotalDifficulty) } - if config.TerminalBlockHash != (common.Hash{}) { if hash := api.eth.BlockChain().GetCanonicalHash(uint64(config.TerminalBlockNumber)); hash == config.TerminalBlockHash { return &beacon.TransitionConfigurationV1{ @@ -257,12 +343,33 @@ func (api *ConsensusAPI) GetPayloadV1(payloadID beacon.PayloadID) (*beacon.Execu // NewPayloadV1 creates an Eth1 block, inserts it in the chain, and returns the status of the chain. func (api *ConsensusAPI) NewPayloadV1(params beacon.ExecutableDataV1) (beacon.PayloadStatusV1, error) { + // The locking here is, strictly, not required. Without these locks, this can happen: + // + // 1. NewPayload( execdata-N ) is invoked from the CL. It goes all the way down to + // api.eth.BlockChain().InsertBlockWithoutSetHead, where it is blocked on + // e.g database compaction. + // 2. The call times out on the CL layer, which issues another NewPayload (execdata-N) call. + // Similarly, this also get stuck on the same place. Importantly, since the + // first call has not gone through, the early checks for "do we already have this block" + // will all return false. + // 3. When the db compaction ends, then N calls inserting the same payload are processed + // sequentially. + // Hence, we use a lock here, to be sure that the previous call has finished before we + // check whether we already have the block locally. + api.newPayloadLock.Lock() + defer api.newPayloadLock.Unlock() + log.Trace("Engine API request received", "method", "ExecutePayload", "number", params.Number, "hash", params.BlockHash) block, err := beacon.ExecutableDataToBlock(params) if err != nil { log.Debug("Invalid NewPayload params", "params", params, "error", err) return beacon.PayloadStatusV1{Status: beacon.INVALIDBLOCKHASH}, nil } + // Stash away the last update to warn the user if the beacon client goes offline + api.lastNewPayloadLock.Lock() + api.lastNewPayloadUpdate = time.Now() + api.lastNewPayloadLock.Unlock() + // If we already have the block locally, ignore the entire execution and just // return a fake success. if block := api.eth.BlockChain().GetBlockByHash(params.BlockHash); block != nil { @@ -270,6 +377,10 @@ func (api *ConsensusAPI) NewPayloadV1(params beacon.ExecutableDataV1) (beacon.Pa hash := block.Hash() return beacon.PayloadStatusV1{Status: beacon.VALID, LatestValidHash: &hash}, nil } + // If this block was rejected previously, keep rejecting it + if res := api.checkInvalidAncestor(block.Hash(), block.Hash()); res != nil { + return *res, nil + } // If the parent is missing, we - in theory - could trigger a sync, but that // would also entail a reorg. That is problematic if multiple sibling blocks // are being fed to us, and even more so, if some semi-distant uncle shortens @@ -278,23 +389,7 @@ func (api *ConsensusAPI) NewPayloadV1(params beacon.ExecutableDataV1) (beacon.Pa // update after legit payload executions. parent := api.eth.BlockChain().GetBlock(block.ParentHash(), block.NumberU64()-1) if parent == nil { - // Stash the block away for a potential forced forckchoice update to it - // at a later time. - api.remoteBlocks.put(block.Hash(), block.Header()) - - // Although we don't want to trigger a sync, if there is one already in - // progress, try to extend if with the current payload request to relieve - // some strain from the forkchoice update. - if err := api.eth.Downloader().BeaconExtend(api.eth.SyncMode(), block.Header()); err == nil { - log.Debug("Payload accepted for sync extension", "number", params.Number, "hash", params.BlockHash) - return beacon.PayloadStatusV1{Status: beacon.SYNCING}, nil - } - // Either no beacon sync was started yet, or it rejected the delivered - // payload as non-integratable on top of the existing sync. We'll just - // have to rely on the beacon client to forcefully update the head with - // a forkchoice update request. - log.Warn("Ignoring payload with missing parent", "number", params.Number, "hash", params.BlockHash, "parent", params.ParentHash) - return beacon.PayloadStatusV1{Status: beacon.ACCEPTED}, nil + return api.delayPayloadImport(block) } // We have an existing parent, do some sanity checks to avoid the beacon client // triggering too early @@ -313,7 +408,14 @@ func (api *ConsensusAPI) NewPayloadV1(params beacon.ExecutableDataV1) (beacon.Pa } if block.Time() <= parent.Time() { log.Warn("Invalid timestamp", "parent", block.Time(), "block", block.Time()) - return api.invalid(errors.New("invalid timestamp"), parent), nil + return api.invalid(errors.New("invalid timestamp"), parent.Header()), nil + } + // Another cornercase: if the node is in snap sync mode, but the CL client + // tries to make it import a block. That should be denied as pushing something + // into the database directly will conflict with the assumptions of snap sync + // that it has an empty db that it can fill itself. + if api.eth.SyncMode() != downloader.FullSync { + return api.delayPayloadImport(block) } if !api.eth.BlockChain().HasBlockAndState(block.ParentHash(), block.NumberU64()-1) { api.remoteBlocks.put(block.Hash(), block.Header()) @@ -323,7 +425,13 @@ func (api *ConsensusAPI) NewPayloadV1(params beacon.ExecutableDataV1) (beacon.Pa log.Trace("Inserting block without sethead", "hash", block.Hash(), "number", block.Number) if err := api.eth.BlockChain().InsertBlockWithoutSetHead(block); err != nil { log.Warn("NewPayloadV1: inserting block failed", "error", err) - return api.invalid(err, parent), nil + + api.invalidLock.Lock() + api.invalidBlocksHits[block.Hash()] = 1 + api.invalidTipsets[block.Hash()] = block.Header() + api.invalidLock.Unlock() + + return api.invalid(err, parent.Header()), nil } // We've accepted a valid payload from the beacon client. Mark the local // chain transitions to notify other subsystems (e.g. downloader) of the @@ -349,14 +457,114 @@ func computePayloadId(headBlockHash common.Hash, params *beacon.PayloadAttribute return out } +// delayPayloadImport stashes the given block away for import at a later time, +// either via a forkchoice update or a sync extension. This method is meant to +// be called by the newpayload command when the block seems to be ok, but some +// prerequisite prevents it from being processed (e.g. no parent, or snap sync). +func (api *ConsensusAPI) delayPayloadImport(block *types.Block) (beacon.PayloadStatusV1, error) { + // Sanity check that this block's parent is not on a previously invalidated + // chain. If it is, mark the block as invalid too. + if res := api.checkInvalidAncestor(block.ParentHash(), block.Hash()); res != nil { + return *res, nil + } + // Stash the block away for a potential forced forkchoice update to it + // at a later time. + api.remoteBlocks.put(block.Hash(), block.Header()) + + // Although we don't want to trigger a sync, if there is one already in + // progress, try to extend if with the current payload request to relieve + // some strain from the forkchoice update. + if err := api.eth.Downloader().BeaconExtend(api.eth.SyncMode(), block.Header()); err == nil { + log.Debug("Payload accepted for sync extension", "number", block.NumberU64(), "hash", block.Hash()) + return beacon.PayloadStatusV1{Status: beacon.SYNCING}, nil + } + // Either no beacon sync was started yet, or it rejected the delivered + // payload as non-integratable on top of the existing sync. We'll just + // have to rely on the beacon client to forcefully update the head with + // a forkchoice update request. + if api.eth.SyncMode() == downloader.FullSync { + // In full sync mode, failure to import a well-formed block can only mean + // that the parent state is missing and the syncer rejected extending the + // current cycle with the new payload. + log.Warn("Ignoring payload with missing parent", "number", block.NumberU64(), "hash", block.Hash(), "parent", block.ParentHash()) + } else { + // In non-full sync mode (i.e. snap sync) all payloads are rejected until + // snap sync terminates as snap sync relies on direct database injections + // and cannot afford concurrent out-if-band modifications via imports. + log.Warn("Ignoring payload while snap syncing", "number", block.NumberU64(), "hash", block.Hash()) + } + return beacon.PayloadStatusV1{Status: beacon.SYNCING}, nil +} + +// setInvalidAncestor is a callback for the downloader to notify us if a bad block +// is encountered during the async sync. +func (api *ConsensusAPI) setInvalidAncestor(invalid *types.Header, origin *types.Header) { + api.invalidLock.Lock() + defer api.invalidLock.Unlock() + + api.invalidTipsets[origin.Hash()] = invalid + api.invalidBlocksHits[invalid.Hash()]++ +} + +// checkInvalidAncestor checks whether the specified chain end links to a known +// bad ancestor. If yes, it constructs the payload failure response to return. +func (api *ConsensusAPI) checkInvalidAncestor(check common.Hash, head common.Hash) *beacon.PayloadStatusV1 { + api.invalidLock.Lock() + defer api.invalidLock.Unlock() + + // If the hash to check is unknown, return valid + invalid, ok := api.invalidTipsets[check] + if !ok { + return nil + } + // If the bad hash was hit too many times, evict it and try to reprocess in + // the hopes that we have a data race that we can exit out of. + badHash := invalid.Hash() + + api.invalidBlocksHits[badHash]++ + if api.invalidBlocksHits[badHash] >= invalidBlockHitEviction { + log.Warn("Too many bad block import attempt, trying", "number", invalid.Number, "hash", badHash) + delete(api.invalidBlocksHits, badHash) + + for descendant, badHeader := range api.invalidTipsets { + if badHeader.Hash() == badHash { + delete(api.invalidTipsets, descendant) + } + } + return nil + } + // Not too many failures yet, mark the head of the invalid chain as invalid + if check != head { + log.Warn("Marked new chain head as invalid", "hash", head, "badnumber", invalid.Number, "badhash", badHash) + for len(api.invalidTipsets) >= invalidTipsetsCap { + for key := range api.invalidTipsets { + delete(api.invalidTipsets, key) + break + } + } + api.invalidTipsets[head] = invalid + } + // If the last valid hash is the terminal pow block, return 0x0 for latest valid hash + lastValid := &invalid.ParentHash + if header := api.eth.BlockChain().GetHeader(invalid.ParentHash, invalid.Number.Uint64()-1); header != nil && header.Difficulty.Sign() != 0 { + lastValid = &common.Hash{} + } + failure := "links to previously rejected block" + return &beacon.PayloadStatusV1{ + Status: beacon.INVALID, + LatestValidHash: lastValid, + ValidationError: &failure, + } +} + // invalid returns a response "INVALID" with the latest valid hash supplied by latest or to the current head // if no latestValid block was provided. -func (api *ConsensusAPI) invalid(err error, latestValid *types.Block) beacon.PayloadStatusV1 { +func (api *ConsensusAPI) invalid(err error, latestValid *types.Header) beacon.PayloadStatusV1 { currentHash := api.eth.BlockChain().CurrentBlock().Hash() if latestValid != nil { // Set latest valid hash to 0x0 if parent is PoW block currentHash = common.Hash{} - if latestValid.Difficulty().BitLen() == 0 { + if latestValid.Difficulty.BitLen() == 0 { // Otherwise set latest valid hash to parent hash currentHash = latestValid.Hash() } @@ -364,3 +572,124 @@ func (api *ConsensusAPI) invalid(err error, latestValid *types.Block) beacon.Pay errorMsg := err.Error() return beacon.PayloadStatusV1{Status: beacon.INVALID, LatestValidHash: ¤tHash, ValidationError: &errorMsg} } + +// heartbeat loops indefinitely, and checks if there have been beacon client updates +// received in the last while. If not - or if they but strange ones - it warns the +// user that something might be off with their consensus node. +// +// TODO(karalabe): Spin this goroutine down somehow +func (api *ConsensusAPI) heartbeat() { + // Sleep a bit on startup since there's obviously no beacon client yet + // attached, so no need to print scary warnings to the user. + time.Sleep(beaconUpdateStartupTimeout) + + var ( + offlineLogged time.Time + ttd = api.eth.BlockChain().Config().TerminalTotalDifficulty + ) + // If the network is not yet merged/merging, don't bother continuing. + if ttd == nil { + return + } + for { + // Sleep a bit and retrieve the last known consensus updates + time.Sleep(5 * time.Second) + + api.lastTransitionLock.Lock() + lastTransitionUpdate := api.lastTransitionUpdate + api.lastTransitionLock.Unlock() + + api.lastForkchoiceLock.Lock() + lastForkchoiceUpdate := api.lastForkchoiceUpdate + api.lastForkchoiceLock.Unlock() + + api.lastNewPayloadLock.Lock() + lastNewPayloadUpdate := api.lastNewPayloadUpdate + api.lastNewPayloadLock.Unlock() + + // If there have been no updates for the past while, warn the user + // that the beacon client is probably offline + if api.eth.BlockChain().Config().TerminalTotalDifficultyPassed || api.eth.Merger().TDDReached() { + if time.Since(lastForkchoiceUpdate) <= beaconUpdateConsensusTimeout || time.Since(lastNewPayloadUpdate) <= beaconUpdateConsensusTimeout { + offlineLogged = time.Time{} + continue + } + if time.Since(lastTransitionUpdate) > beaconUpdateExchangeTimeout { + if time.Since(offlineLogged) > beaconUpdateWarnFrequency { + if lastTransitionUpdate.IsZero() { + log.Warn("Post-merge network, but no beacon client seen. Please launch one to follow the chain!") + } else { + log.Warn("Previously seen beacon client is offline. Please ensure it is operational to follow the chain!") + } + offlineLogged = time.Now() + } + continue + } + if time.Since(offlineLogged) > beaconUpdateWarnFrequency { + if lastForkchoiceUpdate.IsZero() && lastNewPayloadUpdate.IsZero() { + log.Warn("Beacon client online, but never received consensus updates. Please ensure your beacon client is operational to follow the chain!") + } else { + log.Warn("Beacon client online, but no consensus updates received in a while. Please fix your beacon client to follow the chain!") + } + offlineLogged = time.Now() + } + continue + } + if time.Since(lastTransitionUpdate) <= beaconUpdateExchangeTimeout { + offlineLogged = time.Time{} + continue + } + if time.Since(offlineLogged) > beaconUpdateWarnFrequency { + // Retrieve the last few blocks and make a rough estimate as + // to when the merge transition should happen + var ( + chain = api.eth.BlockChain() + head = chain.CurrentHeader() + htd = chain.GetTd(head.Hash(), head.Number.Uint64()) + ) + if htd.Cmp(ttd) >= 0 { + if lastTransitionUpdate.IsZero() { + log.Warn("Merge already reached, but no beacon client seen. Please launch one to follow the chain!") + } else { + log.Warn("Merge already reached, but previously seen beacon client is offline. Please ensure it is operational to follow the chain!") + } + offlineLogged = time.Now() + continue + } + var eta time.Duration + if head.Number.Uint64() > 0 { + // Accumulate the last 64 difficulties to estimate the growth + var ( + deltaDiff uint64 + deltaTime uint64 + current = head + ) + for i := 0; i < 64; i++ { + parent := chain.GetHeader(current.ParentHash, current.Number.Uint64()-1) + if parent == nil { + break + } + deltaDiff += current.Difficulty.Uint64() + deltaTime += current.Time - parent.Time + current = parent + } + // Estimate an ETA based on the block times and the difficulty growth + if deltaTime > 0 { + growth := deltaDiff / deltaTime + left := new(big.Int).Sub(ttd, htd) + eta = time.Duration(new(big.Int).Div(left, new(big.Int).SetUint64(growth+1)).Uint64()) * time.Second + } + } + message := "Merge is configured, but previously seen beacon client is offline. Please ensure it is operational before the transition arrives!" + if lastTransitionUpdate.IsZero() { + message = "Merge is configured, but no beacon client seen. Please ensure you have one available before the transition arrives!" + } + if eta < time.Second { + log.Warn(message) + } else { + log.Warn(message, "eta", common.PrettyAge(time.Now().Add(-eta))) // weird hack, but duration formatted doesn't handle days + } + offlineLogged = time.Now() + } + } +} diff --git a/eth/catalyst/api_test.go b/eth/catalyst/api_test.go index 6171c14c432c8..480a30b52dc55 100644 --- a/eth/catalyst/api_test.go +++ b/eth/catalyst/api_test.go @@ -20,6 +20,7 @@ import ( "bytes" "fmt" "math/big" + "sync" "testing" "time" @@ -28,7 +29,6 @@ import ( "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/beacon" - "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth" @@ -51,10 +51,9 @@ var ( ) func generatePreMergeChain(n int) (*core.Genesis, []*types.Block) { - db := rawdb.NewMemoryDatabase() - config := params.AllEthashProtocolChanges + config := *params.AllEthashProtocolChanges genesis := &core.Genesis{ - Config: config, + Config: &config, Alloc: core.GenesisAlloc{testAddr: {Balance: testBalance}}, ExtraData: []byte("test genesis"), Timestamp: 9000, @@ -65,13 +64,11 @@ func generatePreMergeChain(n int) (*core.Genesis, []*types.Block) { generate := func(i int, g *core.BlockGen) { g.OffsetTime(5) g.SetExtra([]byte("test")) - tx, _ := types.SignTx(types.NewTransaction(testNonce, common.HexToAddress("0x9a9070028361F7AAbeB3f2F2Dc07F82C4a98A02a"), big.NewInt(1), params.TxGas, big.NewInt(params.InitialBaseFee*2), nil), types.LatestSigner(config), testKey) + tx, _ := types.SignTx(types.NewTransaction(testNonce, common.HexToAddress("0x9a9070028361F7AAbeB3f2F2Dc07F82C4a98A02a"), big.NewInt(1), params.TxGas, big.NewInt(params.InitialBaseFee*2), nil), types.LatestSigner(&config), testKey) g.AddTx(tx) testNonce++ } - gblock := genesis.ToBlock(db) - engine := ethash.NewFaker() - blocks, _ := core.GenerateChain(config, gblock, engine, db, n, generate) + _, blocks, _ := core.GenerateChainWithGenesis(genesis, ethash.NewFaker(), n, generate) totalDifficulty := big.NewInt(0) for _, b := range blocks { totalDifficulty.Add(totalDifficulty, b.Difficulty()) @@ -281,10 +278,12 @@ func TestEth2NewBlock(t *testing.T) { t.Fatalf("Failed to convert executable data to block %v", err) } newResp, err := api.NewPayloadV1(*execData) - if err != nil || newResp.Status != "VALID" { + switch { + case err != nil: t.Fatalf("Failed to insert block: %v", err) - } - if ethservice.BlockChain().CurrentBlock().NumberU64() != block.NumberU64()-1 { + case newResp.Status != "VALID": + t.Fatalf("Failed to insert block: %v", newResp.Status) + case ethservice.BlockChain().CurrentBlock().NumberU64() != block.NumberU64()-1: t.Fatalf("Chain head shouldn't be updated") } checkLogEvents(t, newLogCh, rmLogsCh, 0, 0) @@ -296,8 +295,8 @@ func TestEth2NewBlock(t *testing.T) { if _, err := api.ForkchoiceUpdatedV1(fcState, nil); err != nil { t.Fatalf("Failed to insert block: %v", err) } - if ethservice.BlockChain().CurrentBlock().NumberU64() != block.NumberU64() { - t.Fatalf("Chain head should be updated") + if have, want := ethservice.BlockChain().CurrentBlock().NumberU64(), block.NumberU64(); have != want { + t.Fatalf("Chain head should be updated, have %d want %d", have, want) } checkLogEvents(t, newLogCh, rmLogsCh, 1, 0) @@ -403,7 +402,7 @@ func startEthService(t *testing.T, genesis *core.Genesis, blocks []*types.Block) t.Fatal("can't create node:", err) } - ethcfg := ðconfig.Config{Genesis: genesis, Ethash: ethash.Config{PowMode: ethash.ModeFake}, SyncMode: downloader.SnapSync, TrieTimeout: time.Minute, TrieDirtyCache: 256, TrieCleanCache: 256} + ethcfg := ðconfig.Config{Genesis: genesis, Ethash: ethash.Config{PowMode: ethash.ModeFake}, SyncMode: downloader.FullSync, TrieTimeout: time.Minute, TrieDirtyCache: 256, TrieCleanCache: 256} ethservice, err := eth.New(n, ethcfg) if err != nil { t.Fatal("can't create eth service:", err) @@ -415,7 +414,6 @@ func startEthService(t *testing.T, genesis *core.Genesis, blocks []*types.Block) n.Close() t.Fatal("can't import test blocks:", err) } - time.Sleep(500 * time.Millisecond) // give txpool enough time to consume head event ethservice.SetEtherbase(testAddr) ethservice.SetSynced() @@ -523,18 +521,18 @@ func TestExchangeTransitionConfig(t *testing.T) { TestNewPayloadOnInvalidChain sets up a valid chain and tries to feed blocks from an invalid chain to test if latestValidHash (LVH) works correctly. -We set up the following chain where P1 ... Pn and P1'' are valid while +We set up the following chain where P1 ... Pn and P1” are valid while P1' is invalid. We expect (1) The LVH to point to the current inserted payload if it was valid. (2) The LVH to point to the valid parent on an invalid payload (if the parent is available). (3) If the parent is unavailable, the LVH should not be set. -CommonAncestor◄─▲── P1 ◄── P2 ◄─ P3 ◄─ ... ◄─ Pn - │ - └── P1' ◄─ P2' ◄─ P3' ◄─ ... ◄─ Pn' - │ - └── P1'' + CommonAncestor◄─▲── P1 ◄── P2 ◄─ P3 ◄─ ... ◄─ Pn + │ + └── P1' ◄─ P2' ◄─ P3' ◄─ ... ◄─ Pn' + │ + └── P1'' */ func TestNewPayloadOnInvalidChain(t *testing.T) { genesis, preMergeBlocks := generatePreMergeChain(10) @@ -662,8 +660,8 @@ func TestEmptyBlocks(t *testing.T) { if err != nil { t.Fatal(err) } - if status.Status != beacon.ACCEPTED { - t.Errorf("invalid status: expected ACCEPTED got: %v", status.Status) + if status.Status != beacon.SYNCING { + t.Errorf("invalid status: expected SYNCING got: %v", status.Status) } if status.LatestValidHash != nil { t.Fatalf("invalid LVH: got %v wanted nil", status.LatestValidHash) @@ -773,8 +771,8 @@ func TestTrickRemoteBlockCache(t *testing.T) { if err != nil { panic(err) } - if status.Status == beacon.INVALID { - panic("success") + if status.Status == beacon.VALID { + t.Error("invalid status: VALID on an invalid chain") } // Now reorg to the head of the invalid chain resp, err := apiB.ForkchoiceUpdatedV1(beacon.ForkchoiceStateV1{HeadBlockHash: payload.BlockHash, SafeBlockHash: payload.BlockHash, FinalizedBlockHash: payload.ParentHash}, nil) @@ -782,7 +780,7 @@ func TestTrickRemoteBlockCache(t *testing.T) { t.Fatal(err) } if resp.PayloadStatus.Status == beacon.VALID { - t.Errorf("invalid status: expected INVALID got: %v", resp.PayloadStatus.Status) + t.Error("invalid status: VALID on an invalid chain") } time.Sleep(100 * time.Millisecond) } @@ -859,3 +857,102 @@ func TestNewPayloadOnInvalidTerminalBlock(t *testing.T) { t.Fatalf("error sending invalid forkchoice, invalid status: %v", resp.PayloadStatus.Status) } } + +// TestSimultaneousNewBlock does several parallel inserts, both as +// newPayLoad and forkchoiceUpdate. This is to test that the api behaves +// well even of the caller is not being 'serial'. +func TestSimultaneousNewBlock(t *testing.T) { + genesis, preMergeBlocks := generatePreMergeChain(10) + n, ethservice := startEthService(t, genesis, preMergeBlocks) + defer n.Close() + + var ( + api = NewConsensusAPI(ethservice) + parent = preMergeBlocks[len(preMergeBlocks)-1] + ) + for i := 0; i < 10; i++ { + statedb, _ := ethservice.BlockChain().StateAt(parent.Root()) + ethservice.TxPool().AddLocal(types.MustSignNewTx(testKey, types.LatestSigner(ethservice.BlockChain().Config()), + &types.DynamicFeeTx{ + Nonce: statedb.GetNonce(testAddr), + Value: big.NewInt(0), + GasFeeCap: big.NewInt(2 * params.InitialBaseFee), + GasTipCap: big.NewInt(2 * params.InitialBaseFee), + ChainID: genesis.Config.ChainID, + Gas: 1000000, + To: &common.Address{99}, + })) + execData, err := assembleBlock(api, parent.Hash(), &beacon.PayloadAttributesV1{ + Timestamp: parent.Time() + 5, + }) + if err != nil { + t.Fatalf("Failed to create the executable data %v", err) + } + // Insert it 10 times in parallel. Should be ignored. + { + var ( + wg sync.WaitGroup + testErr error + errMu sync.Mutex + ) + wg.Add(10) + for ii := 0; ii < 10; ii++ { + go func() { + defer wg.Done() + if newResp, err := api.NewPayloadV1(*execData); err != nil { + errMu.Lock() + testErr = fmt.Errorf("Failed to insert block: %w", err) + errMu.Unlock() + } else if newResp.Status != "VALID" { + errMu.Lock() + testErr = fmt.Errorf("Failed to insert block: %v", newResp.Status) + errMu.Unlock() + } + }() + } + wg.Wait() + if testErr != nil { + t.Fatal(testErr) + } + } + block, err := beacon.ExecutableDataToBlock(*execData) + if err != nil { + t.Fatalf("Failed to convert executable data to block %v", err) + } + if ethservice.BlockChain().CurrentBlock().NumberU64() != block.NumberU64()-1 { + t.Fatalf("Chain head shouldn't be updated") + } + fcState := beacon.ForkchoiceStateV1{ + HeadBlockHash: block.Hash(), + SafeBlockHash: block.Hash(), + FinalizedBlockHash: block.Hash(), + } + { + var ( + wg sync.WaitGroup + testErr error + errMu sync.Mutex + ) + wg.Add(10) + // Do each FCU 10 times + for ii := 0; ii < 10; ii++ { + go func() { + defer wg.Done() + if _, err := api.ForkchoiceUpdatedV1(fcState, nil); err != nil { + errMu.Lock() + testErr = fmt.Errorf("Failed to insert block: %w", err) + errMu.Unlock() + } + }() + } + wg.Wait() + if testErr != nil { + t.Fatal(testErr) + } + } + if have, want := ethservice.BlockChain().CurrentBlock().NumberU64(), block.NumberU64(); have != want { + t.Fatalf("Chain head should be updated, have %d want %d", have, want) + } + parent = block + } +} diff --git a/eth/discovery.go b/eth/discovery.go deleted file mode 100644 index f7c85b4c5d3ba..0000000000000 --- a/eth/discovery.go +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright 2020 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package eth - -import ( - "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/forkid" - "github.com/ethereum/go-ethereum/p2p/enode" - "github.com/ethereum/go-ethereum/rlp" -) - -// ethEntry is the "eth" ENR entry which advertises eth protocol -// on the discovery network. -type ethEntry struct { - ForkID forkid.ID // Fork identifier per EIP-2124 - - // Ignore additional fields (for forward compatibility). - Rest []rlp.RawValue `rlp:"tail"` -} - -// ENRKey implements enr.Entry. -func (e ethEntry) ENRKey() string { - return "eth" -} - -// startEthEntryUpdate starts the ENR updater loop. -func (eth *Ethereum) startEthEntryUpdate(ln *enode.LocalNode) { - var newHead = make(chan core.ChainHeadEvent, 10) - sub := eth.blockchain.SubscribeChainHeadEvent(newHead) - - go func() { - defer sub.Unsubscribe() - for { - select { - case <-newHead: - ln.Set(eth.currentEthEntry()) - case <-sub.Err(): - // Would be nice to sync with eth.Stop, but there is no - // good way to do that. - return - } - } - }() -} - -func (eth *Ethereum) currentEthEntry() *ethEntry { - return ðEntry{ForkID: forkid.NewID(eth.blockchain.Config(), eth.blockchain.Genesis().Hash(), - eth.blockchain.CurrentHeader().Number.Uint64())} -} diff --git a/eth/downloader/api.go b/eth/downloader/api.go index 2024d23deade6..b3f7113bcde9e 100644 --- a/eth/downloader/api.go +++ b/eth/downloader/api.go @@ -25,21 +25,21 @@ import ( "github.com/ethereum/go-ethereum/rpc" ) -// PublicDownloaderAPI provides an API which gives information about the current synchronisation status. +// DownloaderAPI provides an API which gives information about the current synchronisation status. // It offers only methods that operates on data that can be available to anyone without security risks. -type PublicDownloaderAPI struct { +type DownloaderAPI struct { d *Downloader mux *event.TypeMux installSyncSubscription chan chan interface{} uninstallSyncSubscription chan *uninstallSyncSubscriptionRequest } -// NewPublicDownloaderAPI create a new PublicDownloaderAPI. The API has an internal event loop that +// NewDownloaderAPI create a new DownloaderAPI. The API has an internal event loop that // listens for events from the downloader through the global event mux. In case it receives one of // these events it broadcasts it to all syncing subscriptions that are installed through the // installSyncSubscription channel. -func NewPublicDownloaderAPI(d *Downloader, m *event.TypeMux) *PublicDownloaderAPI { - api := &PublicDownloaderAPI{ +func NewDownloaderAPI(d *Downloader, m *event.TypeMux) *DownloaderAPI { + api := &DownloaderAPI{ d: d, mux: m, installSyncSubscription: make(chan chan interface{}), @@ -53,7 +53,7 @@ func NewPublicDownloaderAPI(d *Downloader, m *event.TypeMux) *PublicDownloaderAP // eventLoop runs a loop until the event mux closes. It will install and uninstall new // sync subscriptions and broadcasts sync status updates to the installed sync subscriptions. -func (api *PublicDownloaderAPI) eventLoop() { +func (api *DownloaderAPI) eventLoop() { var ( sub = api.mux.Subscribe(StartEvent{}, DoneEvent{}, FailedEvent{}) syncSubscriptions = make(map[chan interface{}]struct{}) @@ -90,7 +90,7 @@ func (api *PublicDownloaderAPI) eventLoop() { } // Syncing provides information when this nodes starts synchronising with the Ethereum network and when it's finished. -func (api *PublicDownloaderAPI) Syncing(ctx context.Context) (*rpc.Subscription, error) { +func (api *DownloaderAPI) Syncing(ctx context.Context) (*rpc.Subscription, error) { notifier, supported := rpc.NotifierFromContext(ctx) if !supported { return &rpc.Subscription{}, rpc.ErrNotificationsUnsupported @@ -125,7 +125,7 @@ type SyncingResult struct { Status ethereum.SyncProgress `json:"status"` } -// uninstallSyncSubscriptionRequest uninstalles a syncing subscription in the API event loop. +// uninstallSyncSubscriptionRequest uninstalls a syncing subscription in the API event loop. type uninstallSyncSubscriptionRequest struct { c chan interface{} uninstalled chan interface{} @@ -133,9 +133,9 @@ type uninstallSyncSubscriptionRequest struct { // SyncStatusSubscription represents a syncing subscription. type SyncStatusSubscription struct { - api *PublicDownloaderAPI // register subscription in event loop of this api instance - c chan interface{} // channel where events are broadcasted to - unsubOnce sync.Once // make sure unsubscribe logic is executed once + api *DownloaderAPI // register subscription in event loop of this api instance + c chan interface{} // channel where events are broadcasted to + unsubOnce sync.Once // make sure unsubscribe logic is executed once } // Unsubscribe uninstalls the subscription from the DownloadAPI event loop. @@ -159,8 +159,8 @@ func (s *SyncStatusSubscription) Unsubscribe() { } // SubscribeSyncStatus creates a subscription that will broadcast new synchronisation updates. -// The given channel must receive interface values, the result can either -func (api *PublicDownloaderAPI) SubscribeSyncStatus(status chan interface{}) *SyncStatusSubscription { +// The given channel must receive interface values, the result can either. +func (api *DownloaderAPI) SubscribeSyncStatus(status chan interface{}) *SyncStatusSubscription { api.installSyncSubscription <- status return &SyncStatusSubscription{api: api, c: status} } diff --git a/eth/downloader/beaconsync.go b/eth/downloader/beaconsync.go index 533404f6c9b9b..484a4e20de64a 100644 --- a/eth/downloader/beaconsync.go +++ b/eth/downloader/beaconsync.go @@ -137,6 +137,13 @@ func (b *beaconBackfiller) setMode(mode SyncMode) { b.resume() } +// SetBadBlockCallback sets the callback to run when a bad block is hit by the +// block processor. This method is not thread safe and should be set only once +// on startup before system events are fired. +func (d *Downloader) SetBadBlockCallback(onBadBlock badBlockFn) { + d.badBlock = onBadBlock +} + // BeaconSync is the post-merge version of the chain synchronization, where the // chain is not downloaded from genesis onward, rather from trusted head announces // backwards. @@ -229,7 +236,7 @@ func (d *Downloader) findBeaconAncestor() (uint64, error) { // Binary search to find the ancestor start, end := beaconTail.Number.Uint64()-1, number if number := beaconHead.Number.Uint64(); end > number { - // This shouldn't really happen in a healty network, but if the consensus + // This shouldn't really happen in a healthy network, but if the consensus // clients feeds us a shorter chain as the canonical, we should not attempt // to access non-existent skeleton items. log.Warn("Beacon head lower than local chain", "beacon", number, "local", end) diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index c836fdd4b8cf1..af28d9e82097f 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -85,6 +85,10 @@ var ( // peerDropFn is a callback type for dropping a peer detected as malicious. type peerDropFn func(id string) +// badBlockFn is a callback for the async beacon sync to notify the caller that +// the origin header requested to sync to, produced a chain with a bad block. +type badBlockFn func(invalid *types.Header, origin *types.Header) + // headerTask is a set of downloaded headers to queue along with their precomputed // hashes to avoid constant rehashing. type headerTask struct { @@ -113,6 +117,7 @@ type Downloader struct { // Callbacks dropPeer peerDropFn // Drops a peer for misbehaving + badBlock badBlockFn // Reports a block as rejected by the chain // Status synchroniseMock func(id string, hash common.Hash) error // Replacement for synchronise during testing @@ -131,7 +136,6 @@ type Downloader struct { pivotHeader *types.Header // Pivot block header to dynamically push the syncing state root pivotLock sync.RWMutex // Lock protecting pivot header reads from updates - snapSync bool // Whether to run state sync over the snap protocol SnapSyncer *snap.Syncer // TODO(karalabe): make private! hack for now stateSyncStart chan *stateSync @@ -360,7 +364,7 @@ func (d *Downloader) synchronise(id string, hash common.Hash, td, ttd *big.Int, // The beacon header syncer is async. It will start this synchronization and // will continue doing other tasks. However, if synchronization needs to be // cancelled, the syncer needs to know if we reached the startup point (and - // inited the cancel cannel) or not yet. Make sure that we'll signal even in + // inited the cancel channel) or not yet. Make sure that we'll signal even in // case of a failure. if beaconPing != nil { defer func() { @@ -737,9 +741,11 @@ func (d *Downloader) fetchHead(p *peerConnection) (head *types.Header, pivot *ty // calculateRequestSpan calculates what headers to request from a peer when trying to determine the // common ancestor. // It returns parameters to be used for peer.RequestHeadersByNumber: -// from - starting block number -// count - number of headers to request -// skip - number of headers to skip +// +// from - starting block number +// count - number of headers to request +// skip - number of headers to skip +// // and also returns 'max', the last block which is expected to be returned by the remote peers, // given the (from,count,skip) func calculateRequestSpan(remoteHeight, localHeight uint64) (int64, int, int, uint64) { @@ -1457,7 +1463,7 @@ func (d *Downloader) processHeaders(origin uint64, td, ttd *big.Int, beaconMode } d.syncStatsLock.Unlock() - // Signal the content downloaders of the availablility of new tasks + // Signal the content downloaders of the availability of new tasks for _, ch := range []chan bool{d.queue.blockWakeCh, d.queue.receiptWakeCh} { select { case ch <- true: @@ -1529,7 +1535,7 @@ func (d *Downloader) importBlockResults(results []*fetchResult) error { return errCancelContentProcessing default: } - // Retrieve the a batch of results to import + // Retrieve a batch of results to import first, last := results[0].Header, results[len(results)-1].Header log.Debug("Inserting downloaded chain", "items", len(results), "firstnum", first.Number, "firsthash", first.Hash(), @@ -1545,6 +1551,16 @@ func (d *Downloader) importBlockResults(results []*fetchResult) error { if index, err := d.blockchain.InsertChain(blocks); err != nil { if index < len(results) { log.Debug("Downloaded item processing failed", "number", results[index].Header.Number, "hash", results[index].Header.Hash(), "err", err) + + // In post-merge, notify the engine API of encountered bad chains + if d.badBlock != nil { + head, _, err := d.skeleton.Bounds() + if err != nil { + log.Error("Failed to retrieve beacon bounds for bad block reporting", "err", err) + } else { + d.badBlock(blocks[index].Header(), head) + } + } } else { // The InsertChain method in blockchain.go will sometimes return an out-of-bounds index, // when it needs to preprocess blocks to import a sidechain. diff --git a/eth/downloader/downloader_test.go b/eth/downloader/downloader_test.go index a5db037a456ca..36d6795e7afe7 100644 --- a/eth/downloader/downloader_test.go +++ b/eth/downloader/downloader_test.go @@ -68,9 +68,12 @@ func newTesterWithNotification(t *testing.T, success func()) *downloadTester { t.Cleanup(func() { db.Close() }) - core.GenesisBlockForTesting(db, testAddress, big.NewInt(1000000000000000)) - - chain, err := core.NewBlockChain(db, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil) + gspec := &core.Genesis{ + Config: params.TestChainConfig, + Alloc: core.GenesisAlloc{testAddress: {Balance: big.NewInt(1000000000000000)}}, + BaseFee: big.NewInt(params.InitialBaseFee), + } + chain, err := core.NewBlockChain(db, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) if err != nil { panic(err) } @@ -356,7 +359,7 @@ func (dlp *downloadTesterPeer) RequestAccountRange(id uint64, root, origin, limi } // RequestStorageRanges fetches a batch of storage slots belonging to one or -// more accounts. If slots from only one accout is requested, an origin marker +// more accounts. If slots from only one account is requested, an origin marker // may also be used to retrieve from there. func (dlp *downloadTesterPeer) RequestStorageRanges(id uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, bytes uint64) error { // Create the request and service it @@ -395,7 +398,7 @@ func (dlp *downloadTesterPeer) RequestByteCodes(id uint64, hashes []common.Hash, } // RequestTrieNodes fetches a batch of account or storage trie nodes rooted in -// a specificstate trie. +// a specific state trie. func (dlp *downloadTesterPeer) RequestTrieNodes(id uint64, root common.Hash, paths []snap.TrieNodePathSet, bytes uint64) error { req := &snap.GetTrieNodesPacket{ ID: id, @@ -437,6 +440,9 @@ func assertOwnChain(t *testing.T, tester *downloadTester, length int) { func TestCanonicalSynchronisation66Full(t *testing.T) { testCanonSync(t, eth.ETH66, FullSync) } func TestCanonicalSynchronisation66Snap(t *testing.T) { testCanonSync(t, eth.ETH66, SnapSync) } func TestCanonicalSynchronisation66Light(t *testing.T) { testCanonSync(t, eth.ETH66, LightSync) } +func TestCanonicalSynchronisation67Full(t *testing.T) { testCanonSync(t, eth.ETH67, FullSync) } +func TestCanonicalSynchronisation67Snap(t *testing.T) { testCanonSync(t, eth.ETH67, SnapSync) } +func TestCanonicalSynchronisation67Light(t *testing.T) { testCanonSync(t, eth.ETH67, LightSync) } func testCanonSync(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -457,6 +463,8 @@ func testCanonSync(t *testing.T, protocol uint, mode SyncMode) { // until the cached blocks are retrieved. func TestThrottling66Full(t *testing.T) { testThrottling(t, eth.ETH66, FullSync) } func TestThrottling66Snap(t *testing.T) { testThrottling(t, eth.ETH66, SnapSync) } +func TestThrottling67Full(t *testing.T) { testThrottling(t, eth.ETH67, FullSync) } +func TestThrottling67Snap(t *testing.T) { testThrottling(t, eth.ETH67, SnapSync) } func testThrottling(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -537,6 +545,9 @@ func testThrottling(t *testing.T, protocol uint, mode SyncMode) { func TestForkedSync66Full(t *testing.T) { testForkedSync(t, eth.ETH66, FullSync) } func TestForkedSync66Snap(t *testing.T) { testForkedSync(t, eth.ETH66, SnapSync) } func TestForkedSync66Light(t *testing.T) { testForkedSync(t, eth.ETH66, LightSync) } +func TestForkedSync67Full(t *testing.T) { testForkedSync(t, eth.ETH67, FullSync) } +func TestForkedSync67Snap(t *testing.T) { testForkedSync(t, eth.ETH67, SnapSync) } +func TestForkedSync67Light(t *testing.T) { testForkedSync(t, eth.ETH67, LightSync) } func testForkedSync(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -559,11 +570,14 @@ func testForkedSync(t *testing.T, protocol uint, mode SyncMode) { assertOwnChain(t, tester, len(chainB.blocks)) } -// Tests that synchronising against a much shorter but much heavyer fork works -// corrently and is not dropped. +// Tests that synchronising against a much shorter but much heavier fork works +// currently and is not dropped. func TestHeavyForkedSync66Full(t *testing.T) { testHeavyForkedSync(t, eth.ETH66, FullSync) } func TestHeavyForkedSync66Snap(t *testing.T) { testHeavyForkedSync(t, eth.ETH66, SnapSync) } func TestHeavyForkedSync66Light(t *testing.T) { testHeavyForkedSync(t, eth.ETH66, LightSync) } +func TestHeavyForkedSync67Full(t *testing.T) { testHeavyForkedSync(t, eth.ETH67, FullSync) } +func TestHeavyForkedSync67Snap(t *testing.T) { testHeavyForkedSync(t, eth.ETH67, SnapSync) } +func TestHeavyForkedSync67Light(t *testing.T) { testHeavyForkedSync(t, eth.ETH67, LightSync) } func testHeavyForkedSync(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -593,6 +607,9 @@ func testHeavyForkedSync(t *testing.T, protocol uint, mode SyncMode) { func TestBoundedForkedSync66Full(t *testing.T) { testBoundedForkedSync(t, eth.ETH66, FullSync) } func TestBoundedForkedSync66Snap(t *testing.T) { testBoundedForkedSync(t, eth.ETH66, SnapSync) } func TestBoundedForkedSync66Light(t *testing.T) { testBoundedForkedSync(t, eth.ETH66, LightSync) } +func TestBoundedForkedSync67Full(t *testing.T) { testBoundedForkedSync(t, eth.ETH67, FullSync) } +func TestBoundedForkedSync67Snap(t *testing.T) { testBoundedForkedSync(t, eth.ETH67, SnapSync) } +func TestBoundedForkedSync67Light(t *testing.T) { testBoundedForkedSync(t, eth.ETH67, LightSync) } func testBoundedForkedSync(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -627,6 +644,15 @@ func TestBoundedHeavyForkedSync66Snap(t *testing.T) { func TestBoundedHeavyForkedSync66Light(t *testing.T) { testBoundedHeavyForkedSync(t, eth.ETH66, LightSync) } +func TestBoundedHeavyForkedSync67Full(t *testing.T) { + testBoundedHeavyForkedSync(t, eth.ETH67, FullSync) +} +func TestBoundedHeavyForkedSync67Snap(t *testing.T) { + testBoundedHeavyForkedSync(t, eth.ETH67, SnapSync) +} +func TestBoundedHeavyForkedSync67Light(t *testing.T) { + testBoundedHeavyForkedSync(t, eth.ETH67, LightSync) +} func testBoundedHeavyForkedSync(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -654,6 +680,9 @@ func testBoundedHeavyForkedSync(t *testing.T, protocol uint, mode SyncMode) { func TestCancel66Full(t *testing.T) { testCancel(t, eth.ETH66, FullSync) } func TestCancel66Snap(t *testing.T) { testCancel(t, eth.ETH66, SnapSync) } func TestCancel66Light(t *testing.T) { testCancel(t, eth.ETH66, LightSync) } +func TestCancel67Full(t *testing.T) { testCancel(t, eth.ETH67, FullSync) } +func TestCancel67Snap(t *testing.T) { testCancel(t, eth.ETH67, SnapSync) } +func TestCancel67Light(t *testing.T) { testCancel(t, eth.ETH67, LightSync) } func testCancel(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -681,6 +710,9 @@ func testCancel(t *testing.T, protocol uint, mode SyncMode) { func TestMultiSynchronisation66Full(t *testing.T) { testMultiSynchronisation(t, eth.ETH66, FullSync) } func TestMultiSynchronisation66Snap(t *testing.T) { testMultiSynchronisation(t, eth.ETH66, SnapSync) } func TestMultiSynchronisation66Light(t *testing.T) { testMultiSynchronisation(t, eth.ETH66, LightSync) } +func TestMultiSynchronisation67Full(t *testing.T) { testMultiSynchronisation(t, eth.ETH67, FullSync) } +func TestMultiSynchronisation67Snap(t *testing.T) { testMultiSynchronisation(t, eth.ETH67, SnapSync) } +func TestMultiSynchronisation67Light(t *testing.T) { testMultiSynchronisation(t, eth.ETH67, LightSync) } func testMultiSynchronisation(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -705,6 +737,9 @@ func testMultiSynchronisation(t *testing.T, protocol uint, mode SyncMode) { func TestMultiProtoSynchronisation66Full(t *testing.T) { testMultiProtoSync(t, eth.ETH66, FullSync) } func TestMultiProtoSynchronisation66Snap(t *testing.T) { testMultiProtoSync(t, eth.ETH66, SnapSync) } func TestMultiProtoSynchronisation66Light(t *testing.T) { testMultiProtoSync(t, eth.ETH66, LightSync) } +func TestMultiProtoSynchronisation67Full(t *testing.T) { testMultiProtoSync(t, eth.ETH67, FullSync) } +func TestMultiProtoSynchronisation67Snap(t *testing.T) { testMultiProtoSync(t, eth.ETH67, SnapSync) } +func TestMultiProtoSynchronisation67Light(t *testing.T) { testMultiProtoSync(t, eth.ETH67, LightSync) } func testMultiProtoSync(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -715,7 +750,7 @@ func testMultiProtoSync(t *testing.T, protocol uint, mode SyncMode) { // Create peers of every type tester.newPeer("peer 66", eth.ETH66, chain.blocks[1:]) - //tester.newPeer("peer 65", eth.ETH67, chain.blocks[1:) + tester.newPeer("peer 67", eth.ETH67, chain.blocks[1:]) // Synchronise with the requested peer and make sure all blocks were retrieved if err := tester.sync(fmt.Sprintf("peer %d", protocol), nil, mode); err != nil { @@ -724,7 +759,7 @@ func testMultiProtoSync(t *testing.T, protocol uint, mode SyncMode) { assertOwnChain(t, tester, len(chain.blocks)) // Check that no peers have been dropped off - for _, version := range []int{66} { + for _, version := range []int{66, 67} { peer := fmt.Sprintf("peer %d", version) if _, ok := tester.peers[peer]; !ok { t.Errorf("%s dropped", peer) @@ -737,6 +772,9 @@ func testMultiProtoSync(t *testing.T, protocol uint, mode SyncMode) { func TestEmptyShortCircuit66Full(t *testing.T) { testEmptyShortCircuit(t, eth.ETH66, FullSync) } func TestEmptyShortCircuit66Snap(t *testing.T) { testEmptyShortCircuit(t, eth.ETH66, SnapSync) } func TestEmptyShortCircuit66Light(t *testing.T) { testEmptyShortCircuit(t, eth.ETH66, LightSync) } +func TestEmptyShortCircuit67Full(t *testing.T) { testEmptyShortCircuit(t, eth.ETH67, FullSync) } +func TestEmptyShortCircuit67Snap(t *testing.T) { testEmptyShortCircuit(t, eth.ETH67, SnapSync) } +func TestEmptyShortCircuit67Light(t *testing.T) { testEmptyShortCircuit(t, eth.ETH67, LightSync) } func testEmptyShortCircuit(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -785,6 +823,9 @@ func testEmptyShortCircuit(t *testing.T, protocol uint, mode SyncMode) { func TestMissingHeaderAttack66Full(t *testing.T) { testMissingHeaderAttack(t, eth.ETH66, FullSync) } func TestMissingHeaderAttack66Snap(t *testing.T) { testMissingHeaderAttack(t, eth.ETH66, SnapSync) } func TestMissingHeaderAttack66Light(t *testing.T) { testMissingHeaderAttack(t, eth.ETH66, LightSync) } +func TestMissingHeaderAttack67Full(t *testing.T) { testMissingHeaderAttack(t, eth.ETH67, FullSync) } +func TestMissingHeaderAttack67Snap(t *testing.T) { testMissingHeaderAttack(t, eth.ETH67, SnapSync) } +func TestMissingHeaderAttack67Light(t *testing.T) { testMissingHeaderAttack(t, eth.ETH67, LightSync) } func testMissingHeaderAttack(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -811,6 +852,9 @@ func testMissingHeaderAttack(t *testing.T, protocol uint, mode SyncMode) { func TestShiftedHeaderAttack66Full(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH66, FullSync) } func TestShiftedHeaderAttack66Snap(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH66, SnapSync) } func TestShiftedHeaderAttack66Light(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH66, LightSync) } +func TestShiftedHeaderAttack67Full(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH67, FullSync) } +func TestShiftedHeaderAttack67Snap(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH67, SnapSync) } +func TestShiftedHeaderAttack67Light(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH67, LightSync) } func testShiftedHeaderAttack(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -837,6 +881,7 @@ func testShiftedHeaderAttack(t *testing.T, protocol uint, mode SyncMode) { // for various failure scenarios. Afterwards a full sync is attempted to make // sure no state was corrupted. func TestInvalidHeaderRollback66Snap(t *testing.T) { testInvalidHeaderRollback(t, eth.ETH66, SnapSync) } +func TestInvalidHeaderRollback67Snap(t *testing.T) { testInvalidHeaderRollback(t, eth.ETH67, SnapSync) } func testInvalidHeaderRollback(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -923,6 +968,15 @@ func TestHighTDStarvationAttack66Snap(t *testing.T) { func TestHighTDStarvationAttack66Light(t *testing.T) { testHighTDStarvationAttack(t, eth.ETH66, LightSync) } +func TestHighTDStarvationAttack67Full(t *testing.T) { + testHighTDStarvationAttack(t, eth.ETH67, FullSync) +} +func TestHighTDStarvationAttack67Snap(t *testing.T) { + testHighTDStarvationAttack(t, eth.ETH67, SnapSync) +} +func TestHighTDStarvationAttack67Light(t *testing.T) { + testHighTDStarvationAttack(t, eth.ETH67, LightSync) +} func testHighTDStarvationAttack(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -937,6 +991,7 @@ func testHighTDStarvationAttack(t *testing.T, protocol uint, mode SyncMode) { // Tests that misbehaving peers are disconnected, whilst behaving ones are not. func TestBlockHeaderAttackerDropping66(t *testing.T) { testBlockHeaderAttackerDropping(t, eth.ETH66) } +func TestBlockHeaderAttackerDropping67(t *testing.T) { testBlockHeaderAttackerDropping(t, eth.ETH67) } func testBlockHeaderAttackerDropping(t *testing.T, protocol uint) { // Define the disconnection requirement for individual hash fetch errors @@ -987,6 +1042,9 @@ func testBlockHeaderAttackerDropping(t *testing.T, protocol uint) { func TestSyncProgress66Full(t *testing.T) { testSyncProgress(t, eth.ETH66, FullSync) } func TestSyncProgress66Snap(t *testing.T) { testSyncProgress(t, eth.ETH66, SnapSync) } func TestSyncProgress66Light(t *testing.T) { testSyncProgress(t, eth.ETH66, LightSync) } +func TestSyncProgress67Full(t *testing.T) { testSyncProgress(t, eth.ETH67, FullSync) } +func TestSyncProgress67Snap(t *testing.T) { testSyncProgress(t, eth.ETH67, SnapSync) } +func TestSyncProgress67Light(t *testing.T) { testSyncProgress(t, eth.ETH67, LightSync) } func testSyncProgress(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -1064,6 +1122,9 @@ func checkProgress(t *testing.T, d *Downloader, stage string, want ethereum.Sync func TestForkedSyncProgress66Full(t *testing.T) { testForkedSyncProgress(t, eth.ETH66, FullSync) } func TestForkedSyncProgress66Snap(t *testing.T) { testForkedSyncProgress(t, eth.ETH66, SnapSync) } func TestForkedSyncProgress66Light(t *testing.T) { testForkedSyncProgress(t, eth.ETH66, LightSync) } +func TestForkedSyncProgress67Full(t *testing.T) { testForkedSyncProgress(t, eth.ETH67, FullSync) } +func TestForkedSyncProgress67Snap(t *testing.T) { testForkedSyncProgress(t, eth.ETH67, SnapSync) } +func TestForkedSyncProgress67Light(t *testing.T) { testForkedSyncProgress(t, eth.ETH67, LightSync) } func testForkedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -1135,6 +1196,9 @@ func testForkedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { func TestFailedSyncProgress66Full(t *testing.T) { testFailedSyncProgress(t, eth.ETH66, FullSync) } func TestFailedSyncProgress66Snap(t *testing.T) { testFailedSyncProgress(t, eth.ETH66, SnapSync) } func TestFailedSyncProgress66Light(t *testing.T) { testFailedSyncProgress(t, eth.ETH66, LightSync) } +func TestFailedSyncProgress67Full(t *testing.T) { testFailedSyncProgress(t, eth.ETH67, FullSync) } +func TestFailedSyncProgress67Snap(t *testing.T) { testFailedSyncProgress(t, eth.ETH67, SnapSync) } +func TestFailedSyncProgress67Light(t *testing.T) { testFailedSyncProgress(t, eth.ETH67, LightSync) } func testFailedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -1201,6 +1265,9 @@ func testFailedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { func TestFakedSyncProgress66Full(t *testing.T) { testFakedSyncProgress(t, eth.ETH66, FullSync) } func TestFakedSyncProgress66Snap(t *testing.T) { testFakedSyncProgress(t, eth.ETH66, SnapSync) } func TestFakedSyncProgress66Light(t *testing.T) { testFakedSyncProgress(t, eth.ETH66, LightSync) } +func TestFakedSyncProgress67Full(t *testing.T) { testFakedSyncProgress(t, eth.ETH67, FullSync) } +func TestFakedSyncProgress67Snap(t *testing.T) { testFakedSyncProgress(t, eth.ETH67, SnapSync) } +func TestFakedSyncProgress67Light(t *testing.T) { testFakedSyncProgress(t, eth.ETH67, LightSync) } func testFakedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t) @@ -1347,6 +1414,11 @@ func TestCheckpointEnforcement66Snap(t *testing.T) { testCheckpointEnforcement(t func TestCheckpointEnforcement66Light(t *testing.T) { testCheckpointEnforcement(t, eth.ETH66, LightSync) } +func TestCheckpointEnforcement67Full(t *testing.T) { testCheckpointEnforcement(t, eth.ETH67, FullSync) } +func TestCheckpointEnforcement67Snap(t *testing.T) { testCheckpointEnforcement(t, eth.ETH67, SnapSync) } +func TestCheckpointEnforcement67Light(t *testing.T) { + testCheckpointEnforcement(t, eth.ETH67, LightSync) +} func testCheckpointEnforcement(t *testing.T, protocol uint, mode SyncMode) { // Create a new tester with a particular hard coded checkpoint block diff --git a/eth/downloader/fetchers_concurrent.go b/eth/downloader/fetchers_concurrent.go index a0aa197175a31..44e6aa8f8d88d 100644 --- a/eth/downloader/fetchers_concurrent.go +++ b/eth/downloader/fetchers_concurrent.go @@ -47,7 +47,7 @@ type typedQueue interface { // capacity is responsible for calculating how many items of the abstracted // type a particular peer is estimated to be able to retrieve within the - // alloted round trip time. + // allotted round trip time. capacity(peer *peerConnection, rtt time.Duration) int // updateCapacity is responsible for updating how many items of the abstracted @@ -58,7 +58,7 @@ type typedQueue interface { // from the download queue to the specified peer. reserve(peer *peerConnection, items int) (*fetchRequest, bool, bool) - // unreserve is resposible for removing the current retrieval allocation + // unreserve is responsible for removing the current retrieval allocation // assigned to a specific peer and placing it back into the pool to allow // reassigning to some other peer. unreserve(peer string) int @@ -190,7 +190,7 @@ func (d *Downloader) concurrentFetch(queue typedQueue, beaconMode bool) error { req, err := queue.request(peer, request, responses) if err != nil { // Sending the request failed, which generally means the peer - // was diconnected in between assignment and network send. + // was disconnected in between assignment and network send. // Although all peer removal operations return allocated tasks // to the queue, that is async, and we can do better here by // immediately pushing the unfulfilled requests. diff --git a/eth/downloader/fetchers_concurrent_bodies.go b/eth/downloader/fetchers_concurrent_bodies.go index a8de410323f33..e84206fe99519 100644 --- a/eth/downloader/fetchers_concurrent_bodies.go +++ b/eth/downloader/fetchers_concurrent_bodies.go @@ -41,7 +41,7 @@ func (q *bodyQueue) pending() int { } // capacity is responsible for calculating how many bodies a particular peer is -// estimated to be able to retrieve within the alloted round trip time. +// estimated to be able to retrieve within the allotted round trip time. func (q *bodyQueue) capacity(peer *peerConnection, rtt time.Duration) int { return peer.BodyCapacity(rtt) } @@ -58,7 +58,7 @@ func (q *bodyQueue) reserve(peer *peerConnection, items int) (*fetchRequest, boo return q.queue.ReserveBodies(peer, items) } -// unreserve is resposible for removing the current body retrieval allocation +// unreserve is responsible for removing the current body retrieval allocation // assigned to a specific peer and placing it back into the pool to allow // reassigning to some other peer. func (q *bodyQueue) unreserve(peer string) int { diff --git a/eth/downloader/fetchers_concurrent_headers.go b/eth/downloader/fetchers_concurrent_headers.go index bd3bb3e00bf37..84c7f209865a9 100644 --- a/eth/downloader/fetchers_concurrent_headers.go +++ b/eth/downloader/fetchers_concurrent_headers.go @@ -41,7 +41,7 @@ func (q *headerQueue) pending() int { } // capacity is responsible for calculating how many headers a particular peer is -// estimated to be able to retrieve within the alloted round trip time. +// estimated to be able to retrieve within the allotted round trip time. func (q *headerQueue) capacity(peer *peerConnection, rtt time.Duration) int { return peer.HeaderCapacity(rtt) } @@ -58,7 +58,7 @@ func (q *headerQueue) reserve(peer *peerConnection, items int) (*fetchRequest, b return q.queue.ReserveHeaders(peer, items), false, false } -// unreserve is resposible for removing the current header retrieval allocation +// unreserve is responsible for removing the current header retrieval allocation // assigned to a specific peer and placing it back into the pool to allow // reassigning to some other peer. func (q *headerQueue) unreserve(peer string) int { diff --git a/eth/downloader/fetchers_concurrent_receipts.go b/eth/downloader/fetchers_concurrent_receipts.go index fee2c34101d27..1c853c2184432 100644 --- a/eth/downloader/fetchers_concurrent_receipts.go +++ b/eth/downloader/fetchers_concurrent_receipts.go @@ -28,7 +28,7 @@ import ( // concurrent fetcher and the downloader. type receiptQueue Downloader -// waker returns a notification channel that gets pinged in case more reecipt +// waker returns a notification channel that gets pinged in case more receipt // fetches have been queued up, so the fetcher might assign it to idle peers. func (q *receiptQueue) waker() chan bool { return q.queue.receiptWakeCh @@ -41,7 +41,7 @@ func (q *receiptQueue) pending() int { } // capacity is responsible for calculating how many receipts a particular peer is -// estimated to be able to retrieve within the alloted round trip time. +// estimated to be able to retrieve within the allotted round trip time. func (q *receiptQueue) capacity(peer *peerConnection, rtt time.Duration) int { return peer.ReceiptCapacity(rtt) } @@ -58,7 +58,7 @@ func (q *receiptQueue) reserve(peer *peerConnection, items int) (*fetchRequest, return q.queue.ReserveReceipts(peer, items) } -// unreserve is resposible for removing the current receipt retrieval allocation +// unreserve is responsible for removing the current receipt retrieval allocation // assigned to a specific peer and placing it back into the pool to allow // reassigning to some other peer. func (q *receiptQueue) unreserve(peer string) int { diff --git a/eth/downloader/peer.go b/eth/downloader/peer.go index d74d23e74d557..6b8269495948e 100644 --- a/eth/downloader/peer.go +++ b/eth/downloader/peer.go @@ -237,6 +237,7 @@ func (ps *peerSet) Register(p *peerConnection) error { } p.rates = msgrate.NewTracker(ps.rates.MeanCapacities(), ps.rates.MedianRoundTrip()) if err := ps.rates.Track(p.id, p.rates); err != nil { + ps.lock.Unlock() return err } ps.peers[p.id] = p diff --git a/eth/downloader/queue.go b/eth/downloader/queue.go index ff34d932f0183..ab3ae3d77d0a8 100644 --- a/eth/downloader/queue.go +++ b/eth/downloader/queue.go @@ -480,9 +480,10 @@ func (q *queue) ReserveReceipts(p *peerConnection, count int) (*fetchRequest, bo // to access the queue, so they already need a lock anyway. // // Returns: -// item - the fetchRequest -// progress - whether any progress was made -// throttle - if the caller should throttle for a while +// +// item - the fetchRequest +// progress - whether any progress was made +// throttle - if the caller should throttle for a while func (q *queue) reserveHeaders(p *peerConnection, count int, taskPool map[common.Hash]*types.Header, taskQueue *prque.Prque, pendPool map[string]*fetchRequest, kind uint) (*fetchRequest, bool, bool) { // Short circuit if the pool has been depleted, or if the peer's already @@ -817,7 +818,6 @@ func (q *queue) deliver(id string, taskPool map[common.Hash]*types.Header, reqTimer metrics.Timer, resInMeter metrics.Meter, resDropMeter metrics.Meter, results int, validate func(index int, header *types.Header) error, reconstruct func(index int, result *fetchResult)) (int, error) { - // Short circuit if the data was never requested request := pendPool[id] if request == nil { @@ -857,10 +857,10 @@ func (q *queue) deliver(id string, taskPool map[common.Hash]*types.Header, } for _, header := range request.Headers[:i] { - if res, stale, err := q.resultCache.GetDeliverySlot(header.Number.Uint64()); err == nil { + if res, stale, err := q.resultCache.GetDeliverySlot(header.Number.Uint64()); err == nil && !stale { reconstruct(accepted, res) } else { - // else: betweeen here and above, some other peer filled this result, + // else: between here and above, some other peer filled this result, // or it was indeed a no-op. This should not happen, but if it does it's // not something to panic about log.Error("Delivery stale", "stale", stale, "number", header.Number.Uint64(), "err", err) diff --git a/eth/downloader/queue_test.go b/eth/downloader/queue_test.go index 09b18afe5df5f..8631b27c9275a 100644 --- a/eth/downloader/queue_test.go +++ b/eth/downloader/queue_test.go @@ -27,24 +27,18 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/trie" ) -var ( - testdb = rawdb.NewMemoryDatabase() - genesis = core.GenesisBlockForTesting(testdb, testAddress, big.NewInt(1000000000000000)) -) - // makeChain creates a chain of n blocks starting at and including parent. // the returned hash chain is ordered head->parent. In addition, every 3rd block // contains a transaction and every 5th an uncle to allow testing correct block // reassembly. func makeChain(n int, seed byte, parent *types.Block, empty bool) ([]*types.Block, []types.Receipts) { - blocks, receipts := core.GenerateChain(params.TestChainConfig, parent, ethash.NewFaker(), testdb, n, func(i int, block *core.BlockGen) { + blocks, receipts := core.GenerateChain(params.TestChainConfig, parent, ethash.NewFaker(), testDB, n, func(i int, block *core.BlockGen) { block.SetCoinbase(common.Address{seed}) // Add one tx to every secondblock if !empty && i%2 == 0 { @@ -70,10 +64,10 @@ var emptyChain *chainData func init() { // Create a chain of blocks to import targetBlocks := 128 - blocks, _ := makeChain(targetBlocks, 0, genesis, false) + blocks, _ := makeChain(targetBlocks, 0, testGenesis, false) chain = &chainData{blocks, 0} - blocks, _ = makeChain(targetBlocks, 0, genesis, true) + blocks, _ = makeChain(targetBlocks, 0, testGenesis, true) emptyChain = &chainData{blocks, 0} } @@ -156,7 +150,7 @@ func TestBasics(t *testing.T) { // The second peer should hit throttling if !throttle { - t.Fatalf("should not throttle") + t.Fatalf("should throttle") } // And not get any fetches at all, since it was throttled to begin with if fetchReq != nil { @@ -185,7 +179,6 @@ func TestBasics(t *testing.T) { if got, exp := fetchReq.Headers[0].Number.Uint64(), uint64(1); got != exp { t.Fatalf("expected header %d, got %d", exp, got) } - } if exp, got := q.blockTaskQueue.Size(), numOfBlocks-10; exp != got { t.Errorf("expected block task queue to be %d, got %d", exp, got) @@ -239,7 +232,6 @@ func TestEmptyBlocks(t *testing.T) { if fetchReq != nil { t.Fatal("there should be no body fetch tasks remaining") } - } if q.blockTaskQueue.Size() != numOfBlocks-10 { t.Errorf("expected block task queue to be %d, got %d", numOfBlocks-10, q.blockTaskQueue.Size()) @@ -253,7 +245,7 @@ func TestEmptyBlocks(t *testing.T) { // there should be nothing to fetch, blocks are empty if fetchReq != nil { - t.Fatal("there should be no body fetch tasks remaining") + t.Fatal("there should be no receipt fetch tasks remaining") } } if q.blockTaskQueue.Size() != numOfBlocks-10 { @@ -273,14 +265,13 @@ func TestEmptyBlocks(t *testing.T) { // some more advanced scenarios func XTestDelivery(t *testing.T) { // the outside network, holding blocks - blo, rec := makeChain(128, 0, genesis, false) + blo, rec := makeChain(128, 0, testGenesis, false) world := newNetwork() world.receipts = rec world.chain = blo world.progress(10) if false { log.Root().SetHandler(log.StdoutHandler) - } q := newQueue(10, 10) var wg sync.WaitGroup @@ -315,7 +306,6 @@ func XTestDelivery(t *testing.T) { fmt.Printf("got %d results, %d tot\n", len(res), tot) // Now we can forget about these world.forget(res[len(res)-1].Header.Number.Uint64()) - } }() wg.Add(1) @@ -396,7 +386,6 @@ func XTestDelivery(t *testing.T) { } for i := 0; i < 50; i++ { time.Sleep(2990 * time.Millisecond) - } }() wg.Add(1) @@ -447,10 +436,8 @@ func (n *network) forget(blocknum uint64) { n.chain = n.chain[index:] n.receipts = n.receipts[index:] n.offset = int(blocknum) - } func (n *network) progress(numBlocks int) { - n.lock.Lock() defer n.lock.Unlock() //fmt.Printf("progressing...\n") @@ -458,7 +445,6 @@ func (n *network) progress(numBlocks int) { n.chain = append(n.chain, newBlocks...) n.receipts = append(n.receipts, newR...) n.cond.Broadcast() - } func (n *network) headers(from int) []*types.Header { diff --git a/eth/downloader/resultstore.go b/eth/downloader/resultstore.go index 3162cd6d5b42e..2dcbbe16c916c 100644 --- a/eth/downloader/resultstore.go +++ b/eth/downloader/resultstore.go @@ -71,10 +71,11 @@ func (r *resultStore) SetThrottleThreshold(threshold uint64) uint64 { // wants to reserve headers for fetching. // // It returns the following: -// stale - if true, this item is already passed, and should not be requested again -// throttled - if true, the store is at capacity, this particular header is not prio now -// item - the result to store data into -// err - any error that occurred +// +// stale - if true, this item is already passed, and should not be requested again +// throttled - if true, the store is at capacity, this particular header is not prio now +// item - the result to store data into +// err - any error that occurred func (r *resultStore) AddFetch(header *types.Header, fastSync bool) (stale, throttled bool, item *fetchResult, err error) { r.lock.Lock() defer r.lock.Unlock() diff --git a/eth/downloader/skeleton.go b/eth/downloader/skeleton.go index be4e8fbfc10cd..517b8378c5181 100644 --- a/eth/downloader/skeleton.go +++ b/eth/downloader/skeleton.go @@ -51,7 +51,7 @@ const requestHeaders = 512 // errSyncLinked is an internal helper error to signal that the current sync // cycle linked up to the genesis block, this the skeleton syncer should ping // the backfiller to resume. Since we already have that logic on sync start, -// piggie-back on that instead of 2 entrypoints. +// piggy-back on that instead of 2 entrypoints. var errSyncLinked = errors.New("sync linked") // errSyncMerged is an internal helper error to signal that the current sync @@ -148,7 +148,7 @@ type backfiller interface { // suspend requests the backfiller to abort any running full or snap sync // based on the skeleton chain as it might be invalid. The backfiller should // gracefully handle multiple consecutive suspends without a resume, even - // on initial sartup. + // on initial startup. // // The method should return the last block header that has been successfully // backfilled, or nil if the backfiller was not resumed. @@ -209,7 +209,7 @@ type skeleton struct { headEvents chan *headUpdate // Notification channel for new heads terminate chan chan error // Termination channel to abort sync - terminated chan struct{} // Channel to signal that the syner is dead + terminated chan struct{} // Channel to signal that the syncer is dead // Callback hooks used during testing syncStarting func() // callback triggered after a sync cycle is inited but before started @@ -358,6 +358,7 @@ func (s *skeleton) sync(head *types.Header) (*types.Header, error) { // If the sync is already done, resume the backfiller. When the loop stops, // terminate the backfiller too. linked := len(s.progress.Subchains) == 1 && + rawdb.HasHeader(s.db, s.progress.Subchains[0].Next, s.scratchHead) && rawdb.HasBody(s.db, s.progress.Subchains[0].Next, s.scratchHead) && rawdb.HasReceipts(s.db, s.progress.Subchains[0].Next, s.scratchHead) if linked { @@ -553,7 +554,7 @@ func (s *skeleton) initSync(head *types.Header) { return } } - // Either we've failed to decode the previus state, or there was none. Start + // Either we've failed to decode the previous state, or there was none. Start // a fresh sync with a single subchain represented by the currently sent // chain head. s.progress = &skeletonProgress{ @@ -823,7 +824,7 @@ func (s *skeleton) executeTask(peer *peerConnection, req *headerRequest) { } } -// revertRequests locates all the currently pending reuqests from a particular +// revertRequests locates all the currently pending requests from a particular // peer and reverts them, rescheduling for others to fulfill. func (s *skeleton) revertRequests(peer string) { // Gather the requests first, revertals need the lock too @@ -871,7 +872,7 @@ func (s *skeleton) revertRequest(req *headerRequest) { delete(s.requests, req.id) // Remove the request from the tracked set and mark the task as not-pending, - // ready for resheduling + // ready for rescheduling s.scratchOwners[(s.scratchHead-req.head)/requestHeaders] = "" } @@ -946,12 +947,12 @@ func (s *skeleton) processResponse(res *headerResponse) (linked bool, merged boo // In the case of full sync it would be enough to check for the body, // but even a full syncing node will generate a receipt once block // processing is done, so it's just one more "needless" check. - var ( - hasBody = rawdb.HasBody(s.db, header.ParentHash, header.Number.Uint64()-1) - hasReceipt = rawdb.HasReceipts(s.db, header.ParentHash, header.Number.Uint64()-1) - ) - if hasBody && hasReceipt { - linked = true + // + // The weird cascading checks are done to minimize the database reads. + linked = rawdb.HasHeader(s.db, header.ParentHash, header.Number.Uint64()-1) && + rawdb.HasBody(s.db, header.ParentHash, header.Number.Uint64()-1) && + rawdb.HasReceipts(s.db, header.ParentHash, header.Number.Uint64()-1) + if linked { break } } diff --git a/eth/downloader/skeleton_test.go b/eth/downloader/skeleton_test.go index 836efabebcb85..41373d33a861f 100644 --- a/eth/downloader/skeleton_test.go +++ b/eth/downloader/skeleton_test.go @@ -21,7 +21,6 @@ import ( "errors" "fmt" "math/big" - "os" "sync/atomic" "testing" "time" @@ -54,7 +53,7 @@ func newHookedBackfiller() backfiller { // suspend requests the backfiller to abort any running full or snap sync // based on the skeleton chain as it might be invalid. The backfiller should // gracefully handle multiple consecutive suspends without a resume, even -// on initial sartup. +// on initial startup. func (hf *hookedBackfiller) suspend() *types.Header { if hf.suspendHook != nil { hf.suspendHook() @@ -112,7 +111,7 @@ func newSkeletonTestPeerWithHook(id string, headers []*types.Header, serve func( // function can be used to retrieve batches of headers from the particular peer. func (p *skeletonTestPeer) RequestHeadersByNumber(origin uint64, amount int, skip int, reverse bool, sink chan *eth.Response) (*eth.Request, error) { // Since skeleton test peer are in-memory mocks, dropping the does not make - // them inaccepssible. As such, check a local `dropped` field to see if the + // them inaccessible. As such, check a local `dropped` field to see if the // peer has been dropped and should not respond any more. if atomic.LoadUint64(&p.dropped) != 0 { return nil, errors.New("peer already dropped") @@ -205,7 +204,7 @@ func (p *skeletonTestPeer) RequestReceipts([]common.Hash, chan *eth.Response) (* panic("skeleton sync must not request receipts") } -// Tests various sync initialzations based on previous leftovers in the database +// Tests various sync initializations based on previous leftovers in the database // and announced heads. func TestSkeletonSyncInit(t *testing.T) { // Create a few key headers @@ -228,7 +227,7 @@ func TestSkeletonSyncInit(t *testing.T) { newstate: []*subchain{{Head: 50, Tail: 50}}, }, // Empty database with only the genesis set with a leftover empty sync - // progess. This is a synthetic case, just for the sake of covering things. + // progress. This is a synthetic case, just for the sake of covering things. { oldstate: []*subchain{}, head: block50, @@ -515,7 +514,7 @@ func TestSkeletonSyncExtend(t *testing.T) { // Tests that the skeleton sync correctly retrieves headers from one or more // peers without duplicates or other strange side effects. func TestSkeletonSyncRetrievals(t *testing.T) { - log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true)))) + //log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true)))) // Since skeleton headers don't need to be meaningful, beyond a parent hash // progression, create a long fake chain to test with. @@ -534,13 +533,13 @@ func TestSkeletonSyncRetrievals(t *testing.T) { peers []*skeletonTestPeer // Initial peer set to start the sync with midstate []*subchain // Expected sync state after initial cycle midserve uint64 // Expected number of header retrievals after initial cycle - middrop uint64 // Expectd number of peers dropped after initial cycle + middrop uint64 // Expected number of peers dropped after initial cycle - newHead *types.Header // New header to annount on top of the old one + newHead *types.Header // New header to anoint on top of the old one newPeer *skeletonTestPeer // New peer to join the skeleton syncer endstate []*subchain // Expected sync state after the post-init event endserve uint64 // Expected number of header retrievals after the post-init event - enddrop uint64 // Expectd number of peers dropped after the post-init event + enddrop uint64 // Expected number of peers dropped after the post-init event }{ // Completely empty database with only the genesis set. The sync is expected // to create a single subchain with the requested head. No peers however, so @@ -791,7 +790,6 @@ func TestSkeletonSyncRetrievals(t *testing.T) { check := func() error { if len(progress.Subchains) != len(tt.midstate) { return fmt.Errorf("test %d, mid state: subchain count mismatch: have %d, want %d", i, len(progress.Subchains), len(tt.midstate)) - } for j := 0; j < len(progress.Subchains); j++ { if progress.Subchains[j].Head != tt.midstate[j].Head { @@ -805,7 +803,7 @@ func TestSkeletonSyncRetrievals(t *testing.T) { } waitStart := time.Now() - for waitTime := 20 * time.Millisecond; time.Since(waitStart) < time.Second; waitTime = waitTime * 2 { + for waitTime := 20 * time.Millisecond; time.Since(waitStart) < 2*time.Second; waitTime = waitTime * 2 { time.Sleep(waitTime) // Check the post-init end state if it matches the required results json.Unmarshal(rawdb.ReadSkeletonSyncStatus(db), &progress) @@ -857,7 +855,7 @@ func TestSkeletonSyncRetrievals(t *testing.T) { return nil } waitStart = time.Now() - for waitTime := 20 * time.Millisecond; time.Since(waitStart) < time.Second; waitTime = waitTime * 2 { + for waitTime := 20 * time.Millisecond; time.Since(waitStart) < 2*time.Second; waitTime = waitTime * 2 { time.Sleep(waitTime) // Check the post-init end state if it matches the required results json.Unmarshal(rawdb.ReadSkeletonSyncStatus(db), &progress) diff --git a/eth/downloader/testchain_test.go b/eth/downloader/testchain_test.go index 8b873343cac46..01f81a7b1cde8 100644 --- a/eth/downloader/testchain_test.go +++ b/eth/downloader/testchain_test.go @@ -37,7 +37,13 @@ var ( testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") testAddress = crypto.PubkeyToAddress(testKey.PublicKey) testDB = rawdb.NewMemoryDatabase() - testGenesis = core.GenesisBlockForTesting(testDB, testAddress, big.NewInt(1000000000000000)) + + testGspec = &core.Genesis{ + Config: params.TestChainConfig, + Alloc: core.GenesisAlloc{testAddress: {Balance: big.NewInt(1000000000000000)}}, + BaseFee: big.NewInt(params.InitialBaseFee), + } + testGenesis = testGspec.MustCommit(testDB) ) // The common prefix of all test chains: @@ -154,7 +160,7 @@ func (tc *testChain) copy(newlen int) *testChain { // contains a transaction and every 5th an uncle to allow testing correct block // reassembly. func (tc *testChain) generate(n int, seed byte, parent *types.Block, heavy bool) { - blocks, _ := core.GenerateChain(params.TestChainConfig, parent, ethash.NewFaker(), testDB, n, func(i int, block *core.BlockGen) { + blocks, _ := core.GenerateChain(testGspec.Config, parent, ethash.NewFaker(), testDB, n, func(i int, block *core.BlockGen) { block.SetCoinbase(common.Address{seed}) // If a heavy chain is requested, delay blocks to raise difficulty if heavy { @@ -211,10 +217,7 @@ func newTestBlockchain(blocks []*types.Block) *core.BlockChain { if pregenerated { panic("Requested chain generation outside of init") } - db := rawdb.NewMemoryDatabase() - core.GenesisBlockForTesting(db, testAddress, big.NewInt(1000000000000000)) - - chain, err := core.NewBlockChain(db, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil) + chain, err := core.NewBlockChain(rawdb.NewMemoryDatabase(), nil, testGspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) if err != nil { panic(err) } diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index ca29aad8f0803..a897294175eae 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -83,6 +83,7 @@ var Defaults = Config{ TrieDirtyCache: 256, TrieTimeout: 60 * time.Minute, SnapshotCache: 102, + FilterLogCacheSize: 32, Miner: miner.Config{ GasCeil: 30000000, GasPrice: big.NewInt(params.GWei), @@ -171,6 +172,9 @@ type Config struct { SnapshotCache int Preimages bool + // This is the number of blocks for which logs will be cached in the filter system. + FilterLogCacheSize int + // Mining options Miner miner.Config @@ -196,7 +200,7 @@ type Config struct { RPCEVMTimeout time.Duration // RPCTxFeeCap is the global transaction fee(price * gaslimit) cap for - // send-transction variants. The unit is ether. + // send-transaction variants. The unit is ether. RPCTxFeeCap float64 // Checkpoint is a hardcoded checkpoint which can be nil. @@ -205,21 +209,21 @@ type Config struct { // CheckpointOracle is the configuration for checkpoint oracle. CheckpointOracle *params.CheckpointOracleConfig `toml:",omitempty"` - // Arrow Glacier block override (TODO: remove after the fork) - OverrideArrowGlacier *big.Int `toml:",omitempty"` - // OverrideTerminalTotalDifficulty (TODO: remove after the fork) OverrideTerminalTotalDifficulty *big.Int `toml:",omitempty"` + + // OverrideTerminalTotalDifficultyPassed (TODO: remove after the fork) + OverrideTerminalTotalDifficultyPassed *bool `toml:",omitempty"` } // CreateConsensusEngine creates a consensus engine for the given chain configuration. -func CreateConsensusEngine(stack *node.Node, chainConfig *params.ChainConfig, config *ethash.Config, notify []string, noverify bool, db ethdb.Database) consensus.Engine { +func CreateConsensusEngine(stack *node.Node, ethashConfig *ethash.Config, cliqueConfig *params.CliqueConfig, notify []string, noverify bool, db ethdb.Database) consensus.Engine { // If proof-of-authority is requested, set it up var engine consensus.Engine - if chainConfig.Clique != nil { - engine = clique.New(chainConfig.Clique, db) + if cliqueConfig != nil { + engine = clique.New(cliqueConfig, db) } else { - switch config.PowMode { + switch ethashConfig.PowMode { case ethash.ModeFake: log.Warn("Ethash used in fake mode") case ethash.ModeTest: @@ -228,16 +232,16 @@ func CreateConsensusEngine(stack *node.Node, chainConfig *params.ChainConfig, co log.Warn("Ethash used in shared mode") } engine = ethash.New(ethash.Config{ - PowMode: config.PowMode, - CacheDir: stack.ResolvePath(config.CacheDir), - CachesInMem: config.CachesInMem, - CachesOnDisk: config.CachesOnDisk, - CachesLockMmap: config.CachesLockMmap, - DatasetDir: config.DatasetDir, - DatasetsInMem: config.DatasetsInMem, - DatasetsOnDisk: config.DatasetsOnDisk, - DatasetsLockMmap: config.DatasetsLockMmap, - NotifyFull: config.NotifyFull, + PowMode: ethashConfig.PowMode, + CacheDir: stack.ResolvePath(ethashConfig.CacheDir), + CachesInMem: ethashConfig.CachesInMem, + CachesOnDisk: ethashConfig.CachesOnDisk, + CachesLockMmap: ethashConfig.CachesLockMmap, + DatasetDir: ethashConfig.DatasetDir, + DatasetsInMem: ethashConfig.DatasetsInMem, + DatasetsOnDisk: ethashConfig.DatasetsOnDisk, + DatasetsLockMmap: ethashConfig.DatasetsLockMmap, + NotifyFull: ethashConfig.NotifyFull, }, notify, noverify) engine.(*ethash.Ethash).SetThreads(-1) // Disable CPU mining } diff --git a/eth/ethconfig/gen_config.go b/eth/ethconfig/gen_config.go index 0f43c36ad4472..9c7a04364d201 100644 --- a/eth/ethconfig/gen_config.go +++ b/eth/ethconfig/gen_config.go @@ -18,49 +18,50 @@ import ( // MarshalTOML marshals as TOML. func (c Config) MarshalTOML() (interface{}, error) { type Config struct { - Genesis *core.Genesis `toml:",omitempty"` - NetworkId uint64 - SyncMode downloader.SyncMode - EthDiscoveryURLs []string - SnapDiscoveryURLs []string - NoPruning bool - NoPrefetch bool - TxLookupLimit uint64 `toml:",omitempty"` - RequiredBlocks map[uint64]common.Hash `toml:"-"` - LightServ int `toml:",omitempty"` - LightIngress int `toml:",omitempty"` - LightEgress int `toml:",omitempty"` - LightPeers int `toml:",omitempty"` - LightNoPrune bool `toml:",omitempty"` - LightNoSyncServe bool `toml:",omitempty"` - SyncFromCheckpoint bool `toml:",omitempty"` - UltraLightServers []string `toml:",omitempty"` - UltraLightFraction int `toml:",omitempty"` - UltraLightOnlyAnnounce bool `toml:",omitempty"` - SkipBcVersionCheck bool `toml:"-"` - DatabaseHandles int `toml:"-"` - DatabaseCache int - DatabaseFreezer string - TrieCleanCache int - TrieCleanCacheJournal string `toml:",omitempty"` - TrieCleanCacheRejournal time.Duration `toml:",omitempty"` - TrieDirtyCache int - TrieTimeout time.Duration - SnapshotCache int - Preimages bool - Miner miner.Config - Ethash ethash.Config - TxPool core.TxPoolConfig - GPO gasprice.Config - EnablePreimageRecording bool - DocRoot string `toml:"-"` - RPCGasCap uint64 - RPCEVMTimeout time.Duration - RPCTxFeeCap float64 - Checkpoint *params.TrustedCheckpoint `toml:",omitempty"` - CheckpointOracle *params.CheckpointOracleConfig `toml:",omitempty"` - OverrideArrowGlacier *big.Int `toml:",omitempty"` - OverrideTerminalTotalDifficulty *big.Int `toml:",omitempty"` + Genesis *core.Genesis `toml:",omitempty"` + NetworkId uint64 + SyncMode downloader.SyncMode + EthDiscoveryURLs []string + SnapDiscoveryURLs []string + NoPruning bool + NoPrefetch bool + TxLookupLimit uint64 `toml:",omitempty"` + RequiredBlocks map[uint64]common.Hash `toml:"-"` + LightServ int `toml:",omitempty"` + LightIngress int `toml:",omitempty"` + LightEgress int `toml:",omitempty"` + LightPeers int `toml:",omitempty"` + LightNoPrune bool `toml:",omitempty"` + LightNoSyncServe bool `toml:",omitempty"` + SyncFromCheckpoint bool `toml:",omitempty"` + UltraLightServers []string `toml:",omitempty"` + UltraLightFraction int `toml:",omitempty"` + UltraLightOnlyAnnounce bool `toml:",omitempty"` + SkipBcVersionCheck bool `toml:"-"` + DatabaseHandles int `toml:"-"` + DatabaseCache int + DatabaseFreezer string + TrieCleanCache int + TrieCleanCacheJournal string `toml:",omitempty"` + TrieCleanCacheRejournal time.Duration `toml:",omitempty"` + TrieDirtyCache int + TrieTimeout time.Duration + SnapshotCache int + Preimages bool + FilterLogCacheSize int + Miner miner.Config + Ethash ethash.Config + TxPool core.TxPoolConfig + GPO gasprice.Config + EnablePreimageRecording bool + DocRoot string `toml:"-"` + RPCGasCap uint64 + RPCEVMTimeout time.Duration + RPCTxFeeCap float64 + Checkpoint *params.TrustedCheckpoint `toml:",omitempty"` + CheckpointOracle *params.CheckpointOracleConfig `toml:",omitempty"` + OverrideTerminalTotalDifficulty *big.Int `toml:",omitempty"` + OverrideTerminalTotalDifficultyPassed *bool `toml:",omitempty"` } var enc Config enc.Genesis = c.Genesis @@ -93,6 +94,7 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.TrieTimeout = c.TrieTimeout enc.SnapshotCache = c.SnapshotCache enc.Preimages = c.Preimages + enc.FilterLogCacheSize = c.FilterLogCacheSize enc.Miner = c.Miner enc.Ethash = c.Ethash enc.TxPool = c.TxPool @@ -104,57 +106,58 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.RPCTxFeeCap = c.RPCTxFeeCap enc.Checkpoint = c.Checkpoint enc.CheckpointOracle = c.CheckpointOracle - enc.OverrideArrowGlacier = c.OverrideArrowGlacier enc.OverrideTerminalTotalDifficulty = c.OverrideTerminalTotalDifficulty + enc.OverrideTerminalTotalDifficultyPassed = c.OverrideTerminalTotalDifficultyPassed return &enc, nil } // UnmarshalTOML unmarshals from TOML. func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { type Config struct { - Genesis *core.Genesis `toml:",omitempty"` - NetworkId *uint64 - SyncMode *downloader.SyncMode - EthDiscoveryURLs []string - SnapDiscoveryURLs []string - NoPruning *bool - NoPrefetch *bool - TxLookupLimit *uint64 `toml:",omitempty"` - RequiredBlocks map[uint64]common.Hash `toml:"-"` - LightServ *int `toml:",omitempty"` - LightIngress *int `toml:",omitempty"` - LightEgress *int `toml:",omitempty"` - LightPeers *int `toml:",omitempty"` - LightNoPrune *bool `toml:",omitempty"` - LightNoSyncServe *bool `toml:",omitempty"` - SyncFromCheckpoint *bool `toml:",omitempty"` - UltraLightServers []string `toml:",omitempty"` - UltraLightFraction *int `toml:",omitempty"` - UltraLightOnlyAnnounce *bool `toml:",omitempty"` - SkipBcVersionCheck *bool `toml:"-"` - DatabaseHandles *int `toml:"-"` - DatabaseCache *int - DatabaseFreezer *string - TrieCleanCache *int - TrieCleanCacheJournal *string `toml:",omitempty"` - TrieCleanCacheRejournal *time.Duration `toml:",omitempty"` - TrieDirtyCache *int - TrieTimeout *time.Duration - SnapshotCache *int - Preimages *bool - Miner *miner.Config - Ethash *ethash.Config - TxPool *core.TxPoolConfig - GPO *gasprice.Config - EnablePreimageRecording *bool - DocRoot *string `toml:"-"` - RPCGasCap *uint64 - RPCEVMTimeout *time.Duration - RPCTxFeeCap *float64 - Checkpoint *params.TrustedCheckpoint `toml:",omitempty"` - CheckpointOracle *params.CheckpointOracleConfig `toml:",omitempty"` - OverrideArrowGlacier *big.Int `toml:",omitempty"` - OverrideTerminalTotalDifficulty *big.Int `toml:",omitempty"` + Genesis *core.Genesis `toml:",omitempty"` + NetworkId *uint64 + SyncMode *downloader.SyncMode + EthDiscoveryURLs []string + SnapDiscoveryURLs []string + NoPruning *bool + NoPrefetch *bool + TxLookupLimit *uint64 `toml:",omitempty"` + RequiredBlocks map[uint64]common.Hash `toml:"-"` + LightServ *int `toml:",omitempty"` + LightIngress *int `toml:",omitempty"` + LightEgress *int `toml:",omitempty"` + LightPeers *int `toml:",omitempty"` + LightNoPrune *bool `toml:",omitempty"` + LightNoSyncServe *bool `toml:",omitempty"` + SyncFromCheckpoint *bool `toml:",omitempty"` + UltraLightServers []string `toml:",omitempty"` + UltraLightFraction *int `toml:",omitempty"` + UltraLightOnlyAnnounce *bool `toml:",omitempty"` + SkipBcVersionCheck *bool `toml:"-"` + DatabaseHandles *int `toml:"-"` + DatabaseCache *int + DatabaseFreezer *string + TrieCleanCache *int + TrieCleanCacheJournal *string `toml:",omitempty"` + TrieCleanCacheRejournal *time.Duration `toml:",omitempty"` + TrieDirtyCache *int + TrieTimeout *time.Duration + SnapshotCache *int + Preimages *bool + FilterLogCacheSize *int + Miner *miner.Config + Ethash *ethash.Config + TxPool *core.TxPoolConfig + GPO *gasprice.Config + EnablePreimageRecording *bool + DocRoot *string `toml:"-"` + RPCGasCap *uint64 + RPCEVMTimeout *time.Duration + RPCTxFeeCap *float64 + Checkpoint *params.TrustedCheckpoint `toml:",omitempty"` + CheckpointOracle *params.CheckpointOracleConfig `toml:",omitempty"` + OverrideTerminalTotalDifficulty *big.Int `toml:",omitempty"` + OverrideTerminalTotalDifficultyPassed *bool `toml:",omitempty"` } var dec Config if err := unmarshal(&dec); err != nil { @@ -250,6 +253,9 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.Preimages != nil { c.Preimages = *dec.Preimages } + if dec.FilterLogCacheSize != nil { + c.FilterLogCacheSize = *dec.FilterLogCacheSize + } if dec.Miner != nil { c.Miner = *dec.Miner } @@ -283,11 +289,11 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.CheckpointOracle != nil { c.CheckpointOracle = dec.CheckpointOracle } - if dec.OverrideArrowGlacier != nil { - c.OverrideArrowGlacier = dec.OverrideArrowGlacier - } if dec.OverrideTerminalTotalDifficulty != nil { c.OverrideTerminalTotalDifficulty = dec.OverrideTerminalTotalDifficulty } + if dec.OverrideTerminalTotalDifficultyPassed != nil { + c.OverrideTerminalTotalDifficultyPassed = dec.OverrideTerminalTotalDifficultyPassed + } return nil } diff --git a/eth/fetcher/block_fetcher.go b/eth/fetcher/block_fetcher.go index d75ba3f8e0ee0..bd1a34c83c00d 100644 --- a/eth/fetcher/block_fetcher.go +++ b/eth/fetcher/block_fetcher.go @@ -692,7 +692,6 @@ func (f *BlockFetcher) loop() { } else { f.forgetHash(hash) } - } if matched { task.transactions = append(task.transactions[:i], task.transactions[i+1:]...) diff --git a/eth/fetcher/block_fetcher_test.go b/eth/fetcher/block_fetcher_test.go index 06c61ae55d205..9e5693c02e5a1 100644 --- a/eth/fetcher/block_fetcher_test.go +++ b/eth/fetcher/block_fetcher_test.go @@ -36,10 +36,15 @@ import ( ) var ( - testdb = rawdb.NewMemoryDatabase() - testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - testAddress = crypto.PubkeyToAddress(testKey.PublicKey) - genesis = core.GenesisBlockForTesting(testdb, testAddress, big.NewInt(1000000000000000)) + testdb = rawdb.NewMemoryDatabase() + testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + testAddress = crypto.PubkeyToAddress(testKey.PublicKey) + gspec = &core.Genesis{ + Config: params.TestChainConfig, + Alloc: core.GenesisAlloc{testAddress: {Balance: big.NewInt(1000000000000000)}}, + BaseFee: big.NewInt(params.InitialBaseFee), + } + genesis = gspec.MustCommit(testdb) unknownBlock = types.NewBlock(&types.Header{GasLimit: params.GenesisGasLimit, BaseFee: big.NewInt(params.InitialBaseFee)}, nil, nil, nil, trie.NewStackTrie(nil)) ) @@ -48,7 +53,7 @@ var ( // contains a transaction and every 5th an uncle to allow testing correct block // reassembly. func makeChain(n int, seed byte, parent *types.Block) ([]common.Hash, map[common.Hash]*types.Block) { - blocks, _ := core.GenerateChain(params.TestChainConfig, parent, ethash.NewFaker(), testdb, n, func(i int, block *core.BlockGen) { + blocks, _ := core.GenerateChain(gspec.Config, parent, ethash.NewFaker(), testdb, n, func(i int, block *core.BlockGen) { block.SetCoinbase(common.Address{seed}) // If the block number is multiple of 3, send a bonus transaction to the miner diff --git a/eth/fetcher/tx_fetcher.go b/eth/fetcher/tx_fetcher.go index a23cd24bf106e..677a6422b0110 100644 --- a/eth/fetcher/tx_fetcher.go +++ b/eth/fetcher/tx_fetcher.go @@ -120,7 +120,7 @@ type txDelivery struct { direct bool // Whether this is a direct reply or a broadcast } -// txDrop is the notiication that a peer has disconnected. +// txDrop is the notification that a peer has disconnected. type txDrop struct { peer string } @@ -260,56 +260,74 @@ func (f *TxFetcher) Notify(peer string, hashes []common.Hash) error { // Enqueue imports a batch of received transaction into the transaction pool // and the fetcher. This method may be called by both transaction broadcasts and // direct request replies. The differentiation is important so the fetcher can -// re-shedule missing transactions as soon as possible. +// re-schedule missing transactions as soon as possible. func (f *TxFetcher) Enqueue(peer string, txs []*types.Transaction, direct bool) error { - // Keep track of all the propagated transactions - if direct { - txReplyInMeter.Mark(int64(len(txs))) - } else { - txBroadcastInMeter.Mark(int64(len(txs))) + var ( + inMeter = txReplyInMeter + knownMeter = txReplyKnownMeter + underpricedMeter = txReplyUnderpricedMeter + otherRejectMeter = txReplyOtherRejectMeter + ) + if !direct { + inMeter = txBroadcastInMeter + knownMeter = txBroadcastKnownMeter + underpricedMeter = txBroadcastUnderpricedMeter + otherRejectMeter = txBroadcastOtherRejectMeter } + // Keep track of all the propagated transactions + inMeter.Mark(int64(len(txs))) + // Push all the transactions into the pool, tracking underpriced ones to avoid // re-requesting them and dropping the peer in case of malicious transfers. var ( - added = make([]common.Hash, 0, len(txs)) - duplicate int64 - underpriced int64 - otherreject int64 + added = make([]common.Hash, 0, len(txs)) ) - errs := f.addTxs(txs) - for i, err := range errs { - // Track the transaction hash if the price is too low for us. - // Avoid re-request this transaction when we receive another - // announcement. - if errors.Is(err, core.ErrUnderpriced) || errors.Is(err, core.ErrReplaceUnderpriced) { - for f.underpriced.Cardinality() >= maxTxUnderpricedSetSize { - f.underpriced.Pop() - } - f.underpriced.Add(txs[i].Hash()) + // proceed in batches + for i := 0; i < len(txs); i += 128 { + end := i + 128 + if end > len(txs) { + end = len(txs) } - // Track a few interesting failure types - switch { - case err == nil: // Noop, but need to handle to not count these + var ( + duplicate int64 + underpriced int64 + otherreject int64 + ) + batch := txs[i:end] + for j, err := range f.addTxs(batch) { + // Track the transaction hash if the price is too low for us. + // Avoid re-request this transaction when we receive another + // announcement. + if errors.Is(err, core.ErrUnderpriced) || errors.Is(err, core.ErrReplaceUnderpriced) { + for f.underpriced.Cardinality() >= maxTxUnderpricedSetSize { + f.underpriced.Pop() + } + f.underpriced.Add(batch[j].Hash()) + } + // Track a few interesting failure types + switch { + case err == nil: // Noop, but need to handle to not count these - case errors.Is(err, core.ErrAlreadyKnown): - duplicate++ + case errors.Is(err, core.ErrAlreadyKnown): + duplicate++ - case errors.Is(err, core.ErrUnderpriced) || errors.Is(err, core.ErrReplaceUnderpriced): - underpriced++ + case errors.Is(err, core.ErrUnderpriced) || errors.Is(err, core.ErrReplaceUnderpriced): + underpriced++ - default: - otherreject++ + default: + otherreject++ + } + added = append(added, batch[j].Hash()) + } + knownMeter.Mark(duplicate) + underpricedMeter.Mark(underpriced) + otherRejectMeter.Mark(otherreject) + + // If 'other reject' is >25% of the deliveries in any batch, sleep a bit. + if otherreject > 128/4 { + time.Sleep(200 * time.Millisecond) + log.Warn("Peer delivering stale transactions", "peer", peer, "rejected", otherreject) } - added = append(added, txs[i].Hash()) - } - if direct { - txReplyKnownMeter.Mark(duplicate) - txReplyUnderpricedMeter.Mark(underpriced) - txReplyOtherRejectMeter.Mark(otherreject) - } else { - txBroadcastKnownMeter.Mark(duplicate) - txBroadcastUnderpricedMeter.Mark(underpriced) - txBroadcastOtherRejectMeter.Mark(otherreject) } select { case f.cleanup <- &txDelivery{origin: peer, hashes: added, direct: direct}: @@ -558,7 +576,7 @@ func (f *TxFetcher) loop() { // In case of a direct delivery, also reschedule anything missing // from the original query if delivery.direct { - // Mark the reqesting successful (independent of individual status) + // Mark the requesting successful (independent of individual status) txRequestDoneMeter.Mark(int64(len(delivery.hashes))) // Make sure something was pending, nuke it @@ -607,7 +625,7 @@ func (f *TxFetcher) loop() { delete(f.alternates, hash) delete(f.fetching, hash) } - // Something was delivered, try to rechedule requests + // Something was delivered, try to reschedule requests f.scheduleFetches(timeoutTimer, timeoutTrigger, nil) // Partial delivery may enable others to deliver too } @@ -719,7 +737,7 @@ func (f *TxFetcher) rescheduleWait(timer *mclock.Timer, trigger chan struct{}) { // should be rescheduled if some request is pending. In practice, a timeout will // cause the timer to be rescheduled every 5 secs (until the peer comes through or // disconnects). This is a limitation of the fetcher code because we don't trac -// pending requests and timed out requests separatey. Without double tracking, if +// pending requests and timed out requests separately. Without double tracking, if // we simply didn't reschedule the timer on all-timeout then the timer would never // be set again since len(request) > 0 => something's running. func (f *TxFetcher) rescheduleTimeout(timer *mclock.Timer, trigger chan struct{}) { diff --git a/eth/fetcher/tx_fetcher_test.go b/eth/fetcher/tx_fetcher_test.go index ce8d02af7ddff..4c06712f7759d 100644 --- a/eth/fetcher/tx_fetcher_test.go +++ b/eth/fetcher/tx_fetcher_test.go @@ -1011,7 +1011,7 @@ func TestTransactionFetcherOutOfBoundDeliveries(t *testing.T) { } // Tests that dropping a peer cleans out all internal data structures in all the -// live or danglng stages. +// live or dangling stages. func TestTransactionFetcherDrop(t *testing.T) { testTransactionFetcherParallel(t, txFetcherTest{ init: func() *TxFetcher { @@ -1121,7 +1121,7 @@ func TestTransactionFetcherDropRescheduling(t *testing.T) { } // This test reproduces a crash caught by the fuzzer. The root cause was a -// dangling transaction timing out and clashing on readd with a concurrently +// dangling transaction timing out and clashing on re-add with a concurrently // announced one. func TestTransactionFetcherFuzzCrash01(t *testing.T) { testTransactionFetcherParallel(t, txFetcherTest{ @@ -1148,7 +1148,7 @@ func TestTransactionFetcherFuzzCrash01(t *testing.T) { } // This test reproduces a crash caught by the fuzzer. The root cause was a -// dangling transaction getting peer-dropped and clashing on readd with a +// dangling transaction getting peer-dropped and clashing on re-add with a // concurrently announced one. func TestTransactionFetcherFuzzCrash02(t *testing.T) { testTransactionFetcherParallel(t, txFetcherTest{ diff --git a/eth/filters/api.go b/eth/filters/api.go index 7196e90f9eb71..43e63d5ba98a8 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -36,39 +36,39 @@ import ( // and associated subscription in the event system. type filter struct { typ Type - deadline *time.Timer // filter is inactiv when deadline triggers + deadline *time.Timer // filter is inactive when deadline triggers hashes []common.Hash crit FilterCriteria logs []*types.Log s *Subscription // associated subscription in event system } -// PublicFilterAPI offers support to create and manage filters. This will allow external clients to retrieve various +// FilterAPI offers support to create and manage filters. This will allow external clients to retrieve various // information related to the Ethereum protocol such als blocks, transactions and logs. -type PublicFilterAPI struct { - backend Backend +type FilterAPI struct { + sys *FilterSystem events *EventSystem filtersMu sync.Mutex filters map[rpc.ID]*filter timeout time.Duration } -// NewPublicFilterAPI returns a new PublicFilterAPI instance. -func NewPublicFilterAPI(backend Backend, lightMode bool, timeout time.Duration) *PublicFilterAPI { - api := &PublicFilterAPI{ - backend: backend, - events: NewEventSystem(backend, lightMode), +// NewFilterAPI returns a new FilterAPI instance. +func NewFilterAPI(system *FilterSystem, lightMode bool) *FilterAPI { + api := &FilterAPI{ + sys: system, + events: NewEventSystem(system, lightMode), filters: make(map[rpc.ID]*filter), - timeout: timeout, + timeout: system.cfg.Timeout, } - go api.timeoutLoop(timeout) + go api.timeoutLoop(system.cfg.Timeout) return api } // timeoutLoop runs at the interval set by 'timeout' and deletes filters // that have not been recently used. It is started when the API is created. -func (api *PublicFilterAPI) timeoutLoop(timeout time.Duration) { +func (api *FilterAPI) timeoutLoop(timeout time.Duration) { var toUninstall []*Subscription ticker := time.NewTicker(timeout) defer ticker.Stop() @@ -101,9 +101,7 @@ func (api *PublicFilterAPI) timeoutLoop(timeout time.Duration) { // // It is part of the filter package because this filter can be used through the // `eth_getFilterChanges` polling method that is also used for log filters. -// -// https://eth.wiki/json-rpc/API#eth_newpendingtransactionfilter -func (api *PublicFilterAPI) NewPendingTransactionFilter() rpc.ID { +func (api *FilterAPI) NewPendingTransactionFilter() rpc.ID { var ( pendingTxs = make(chan []common.Hash) pendingTxSub = api.events.SubscribePendingTxs(pendingTxs) @@ -136,7 +134,7 @@ func (api *PublicFilterAPI) NewPendingTransactionFilter() rpc.ID { // NewPendingTransactions creates a subscription that is triggered each time a transaction // enters the transaction pool and was signed from one of the transactions this nodes manages. -func (api *PublicFilterAPI) NewPendingTransactions(ctx context.Context) (*rpc.Subscription, error) { +func (api *FilterAPI) NewPendingTransactions(ctx context.Context) (*rpc.Subscription, error) { notifier, supported := rpc.NotifierFromContext(ctx) if !supported { return &rpc.Subscription{}, rpc.ErrNotificationsUnsupported @@ -171,9 +169,7 @@ func (api *PublicFilterAPI) NewPendingTransactions(ctx context.Context) (*rpc.Su // NewBlockFilter creates a filter that fetches blocks that are imported into the chain. // It is part of the filter package since polling goes with eth_getFilterChanges. -// -// https://eth.wiki/json-rpc/API#eth_newblockfilter -func (api *PublicFilterAPI) NewBlockFilter() rpc.ID { +func (api *FilterAPI) NewBlockFilter() rpc.ID { var ( headers = make(chan *types.Header) headerSub = api.events.SubscribeNewHeads(headers) @@ -205,7 +201,7 @@ func (api *PublicFilterAPI) NewBlockFilter() rpc.ID { } // NewHeads send a notification each time a new (header) block is appended to the chain. -func (api *PublicFilterAPI) NewHeads(ctx context.Context) (*rpc.Subscription, error) { +func (api *FilterAPI) NewHeads(ctx context.Context) (*rpc.Subscription, error) { notifier, supported := rpc.NotifierFromContext(ctx) if !supported { return &rpc.Subscription{}, rpc.ErrNotificationsUnsupported @@ -235,7 +231,7 @@ func (api *PublicFilterAPI) NewHeads(ctx context.Context) (*rpc.Subscription, er } // Logs creates a subscription that fires for all new log that match the given filter criteria. -func (api *PublicFilterAPI) Logs(ctx context.Context, crit FilterCriteria) (*rpc.Subscription, error) { +func (api *FilterAPI) Logs(ctx context.Context, crit FilterCriteria) (*rpc.Subscription, error) { notifier, supported := rpc.NotifierFromContext(ctx) if !supported { return &rpc.Subscription{}, rpc.ErrNotificationsUnsupported @@ -252,11 +248,11 @@ func (api *PublicFilterAPI) Logs(ctx context.Context, crit FilterCriteria) (*rpc } go func() { - for { select { case logs := <-matchedLogs: for _, log := range logs { + log := log notifier.Notify(rpcSub.ID, &log) } case <-rpcSub.Err(): // client send an unsubscribe request @@ -287,9 +283,7 @@ type FilterCriteria ethereum.FilterQuery // again but with the removed property set to true. // // In case "fromBlock" > "toBlock" an error is returned. -// -// https://eth.wiki/json-rpc/API#eth_newfilter -func (api *PublicFilterAPI) NewFilter(crit FilterCriteria) (rpc.ID, error) { +func (api *FilterAPI) NewFilter(crit FilterCriteria) (rpc.ID, error) { logs := make(chan []*types.Log) logsSub, err := api.events.SubscribeLogs(ethereum.FilterQuery(crit), logs) if err != nil { @@ -322,13 +316,11 @@ func (api *PublicFilterAPI) NewFilter(crit FilterCriteria) (rpc.ID, error) { } // GetLogs returns logs matching the given argument that are stored within the state. -// -// https://eth.wiki/json-rpc/API#eth_getlogs -func (api *PublicFilterAPI) GetLogs(ctx context.Context, crit FilterCriteria) ([]*types.Log, error) { +func (api *FilterAPI) GetLogs(ctx context.Context, crit FilterCriteria) ([]*types.Log, error) { var filter *Filter if crit.BlockHash != nil { // Block filter requested, construct a single-shot filter - filter = NewBlockFilter(api.backend, *crit.BlockHash, crit.Addresses, crit.Topics) + filter = api.sys.NewBlockFilter(*crit.BlockHash, crit.Addresses, crit.Topics) } else { // Convert the RPC block numbers into internal representations begin := rpc.LatestBlockNumber.Int64() @@ -340,7 +332,7 @@ func (api *PublicFilterAPI) GetLogs(ctx context.Context, crit FilterCriteria) ([ end = crit.ToBlock.Int64() } // Construct the range filter - filter = NewRangeFilter(api.backend, begin, end, crit.Addresses, crit.Topics) + filter = api.sys.NewRangeFilter(begin, end, crit.Addresses, crit.Topics) } // Run the filter and return all the logs logs, err := filter.Logs(ctx) @@ -351,9 +343,7 @@ func (api *PublicFilterAPI) GetLogs(ctx context.Context, crit FilterCriteria) ([ } // UninstallFilter removes the filter with the given filter id. -// -// https://eth.wiki/json-rpc/API#eth_uninstallfilter -func (api *PublicFilterAPI) UninstallFilter(id rpc.ID) bool { +func (api *FilterAPI) UninstallFilter(id rpc.ID) bool { api.filtersMu.Lock() f, found := api.filters[id] if found { @@ -369,9 +359,7 @@ func (api *PublicFilterAPI) UninstallFilter(id rpc.ID) bool { // GetFilterLogs returns the logs for the filter with the given id. // If the filter could not be found an empty array of logs is returned. -// -// https://eth.wiki/json-rpc/API#eth_getfilterlogs -func (api *PublicFilterAPI) GetFilterLogs(ctx context.Context, id rpc.ID) ([]*types.Log, error) { +func (api *FilterAPI) GetFilterLogs(ctx context.Context, id rpc.ID) ([]*types.Log, error) { api.filtersMu.Lock() f, found := api.filters[id] api.filtersMu.Unlock() @@ -383,7 +371,7 @@ func (api *PublicFilterAPI) GetFilterLogs(ctx context.Context, id rpc.ID) ([]*ty var filter *Filter if f.crit.BlockHash != nil { // Block filter requested, construct a single-shot filter - filter = NewBlockFilter(api.backend, *f.crit.BlockHash, f.crit.Addresses, f.crit.Topics) + filter = api.sys.NewBlockFilter(*f.crit.BlockHash, f.crit.Addresses, f.crit.Topics) } else { // Convert the RPC block numbers into internal representations begin := rpc.LatestBlockNumber.Int64() @@ -395,7 +383,7 @@ func (api *PublicFilterAPI) GetFilterLogs(ctx context.Context, id rpc.ID) ([]*ty end = f.crit.ToBlock.Int64() } // Construct the range filter - filter = NewRangeFilter(api.backend, begin, end, f.crit.Addresses, f.crit.Topics) + filter = api.sys.NewRangeFilter(begin, end, f.crit.Addresses, f.crit.Topics) } // Run the filter and return all the logs logs, err := filter.Logs(ctx) @@ -410,9 +398,7 @@ func (api *PublicFilterAPI) GetFilterLogs(ctx context.Context, id rpc.ID) ([]*ty // // For pending transaction and block filters the result is []common.Hash. // (pending)Log filters return []Log. -// -// https://eth.wiki/json-rpc/API#eth_getfilterchanges -func (api *PublicFilterAPI) GetFilterChanges(id rpc.ID) (interface{}, error) { +func (api *FilterAPI) GetFilterChanges(id rpc.ID) (interface{}, error) { api.filtersMu.Lock() defer api.filtersMu.Unlock() diff --git a/eth/filters/api_test.go b/eth/filters/api_test.go index 02229a7549a78..0a80d0f8ddbde 100644 --- a/eth/filters/api_test.go +++ b/eth/filters/api_test.go @@ -56,7 +56,7 @@ func TestUnmarshalJSONNewFilterArgs(t *testing.T) { // from, to block number var test1 FilterCriteria - vector := fmt.Sprintf(`{"fromBlock":"0x%x","toBlock":"0x%x"}`, fromBlock, toBlock) + vector := fmt.Sprintf(`{"fromBlock":"%#x","toBlock":"%#x"}`, fromBlock, toBlock) if err := json.Unmarshal([]byte(vector), &test1); err != nil { t.Fatal(err) } diff --git a/eth/filters/bench_test.go b/eth/filters/bench_test.go index 9632f4195f4cf..73b96b77af62d 100644 --- a/eth/filters/bench_test.go +++ b/eth/filters/bench_test.go @@ -93,9 +93,9 @@ func benchmarkBloomBits(b *testing.B, sectionSize uint64) { var header *types.Header for i := sectionIdx * sectionSize; i < (sectionIdx+1)*sectionSize; i++ { hash := rawdb.ReadCanonicalHash(db, i) - header = rawdb.ReadHeader(db, hash, i) - if header == nil { + if header = rawdb.ReadHeader(db, hash, i); header == nil { b.Fatalf("Error creating bloomBits data") + return } bc.AddBloom(uint(i-sectionIdx*sectionSize), header.Bloom) } @@ -122,31 +122,36 @@ func benchmarkBloomBits(b *testing.B, sectionSize uint64) { b.Log("Running filter benchmarks...") start = time.Now() - var backend *testBackend + var ( + backend *testBackend + sys *FilterSystem + ) for i := 0; i < benchFilterCnt; i++ { if i%20 == 0 { db.Close() db, _ = rawdb.NewLevelDBDatabase(benchDataDir, 128, 1024, "", false) backend = &testBackend{db: db, sections: cnt} + sys = NewFilterSystem(backend, Config{}) } var addr common.Address addr[0] = byte(i) addr[1] = byte(i / 256) - filter := NewRangeFilter(backend, 0, int64(cnt*sectionSize-1), []common.Address{addr}, nil) + filter := sys.NewRangeFilter(0, int64(cnt*sectionSize-1), []common.Address{addr}, nil) if _, err := filter.Logs(context.Background()); err != nil { - b.Error("filter.Find error:", err) + b.Error("filter.Logs error:", err) } } + d = time.Since(start) b.Log("Finished running filter benchmarks") b.Log(" ", d, "total ", d/time.Duration(benchFilterCnt), "per address", d*time.Duration(1000000)/time.Duration(benchFilterCnt*cnt*sectionSize), "per million blocks") db.Close() } -var bloomBitsPrefix = []byte("bloomBits-") - +//nolint:unused func clearBloomBits(db ethdb.Database) { + var bloomBitsPrefix = []byte("bloomBits-") fmt.Println("Clearing bloombits data...") it := db.NewIterator(bloomBitsPrefix, nil) for it.Next() { @@ -171,10 +176,11 @@ func BenchmarkNoBloomBits(b *testing.B) { clearBloomBits(db) + _, sys := newTestFilterSystem(b, db, Config{}) + b.Log("Running filter benchmarks...") start := time.Now() - backend := &testBackend{db: db} - filter := NewRangeFilter(backend, 0, int64(*headNum), []common.Address{{}}, nil) + filter := sys.NewRangeFilter(0, int64(*headNum), []common.Address{{}}, nil) filter.Logs(context.Background()) d := time.Since(start) b.Log("Finished running filter benchmarks") diff --git a/eth/filters/filter.go b/eth/filters/filter.go index f64e84abb86c9..0a70c9ece1dbe 100644 --- a/eth/filters/filter.go +++ b/eth/filters/filter.go @@ -22,36 +22,15 @@ import ( "math/big" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/bloombits" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/rpc" ) -type Backend interface { - ChainDb() ethdb.Database - HeaderByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*types.Header, error) - HeaderByHash(ctx context.Context, blockHash common.Hash) (*types.Header, error) - GetReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error) - GetLogs(ctx context.Context, blockHash common.Hash) ([][]*types.Log, error) - - SubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscription - SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription - SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription - SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription - SubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription - - BloomStatus() (uint64, uint64) - ServiceFilter(ctx context.Context, session *bloombits.MatcherSession) -} - // Filter can be used to retrieve and filter logs. type Filter struct { - backend Backend + sys *FilterSystem - db ethdb.Database addresses []common.Address topics [][]common.Hash @@ -63,7 +42,7 @@ type Filter struct { // NewRangeFilter creates a new filter which uses a bloom filter on blocks to // figure out whether a particular block is interesting or not. -func NewRangeFilter(backend Backend, begin, end int64, addresses []common.Address, topics [][]common.Hash) *Filter { +func (sys *FilterSystem) NewRangeFilter(begin, end int64, addresses []common.Address, topics [][]common.Hash) *Filter { // Flatten the address and topic filter clauses into a single bloombits filter // system. Since the bloombits are not positional, nil topics are permitted, // which get flattened into a nil byte slice. @@ -82,10 +61,10 @@ func NewRangeFilter(backend Backend, begin, end int64, addresses []common.Addres } filters = append(filters, filter) } - size, _ := backend.BloomStatus() + size, _ := sys.backend.BloomStatus() // Create a generic filter and convert it into a range filter - filter := newFilter(backend, addresses, topics) + filter := newFilter(sys, addresses, topics) filter.matcher = bloombits.NewMatcher(size, filters) filter.begin = begin @@ -96,21 +75,20 @@ func NewRangeFilter(backend Backend, begin, end int64, addresses []common.Addres // NewBlockFilter creates a new filter which directly inspects the contents of // a block to figure out whether it is interesting or not. -func NewBlockFilter(backend Backend, block common.Hash, addresses []common.Address, topics [][]common.Hash) *Filter { +func (sys *FilterSystem) NewBlockFilter(block common.Hash, addresses []common.Address, topics [][]common.Hash) *Filter { // Create a generic filter and convert it into a block filter - filter := newFilter(backend, addresses, topics) + filter := newFilter(sys, addresses, topics) filter.block = block return filter } // newFilter creates a generic filter that can either filter based on a block hash, // or based on range queries. The search criteria needs to be explicitly set. -func newFilter(backend Backend, addresses []common.Address, topics [][]common.Hash) *Filter { +func newFilter(sys *FilterSystem, addresses []common.Address, topics [][]common.Hash) *Filter { return &Filter{ - backend: backend, + sys: sys, addresses: addresses, topics: topics, - db: backend.ChainDb(), } } @@ -119,35 +97,44 @@ func newFilter(backend Backend, addresses []common.Address, topics [][]common.Ha func (f *Filter) Logs(ctx context.Context) ([]*types.Log, error) { // If we're doing singleton block filtering, execute and return if f.block != (common.Hash{}) { - header, err := f.backend.HeaderByHash(ctx, f.block) + header, err := f.sys.backend.HeaderByHash(ctx, f.block) if err != nil { return nil, err } if header == nil { return nil, errors.New("unknown block") } - return f.blockLogs(ctx, header) + return f.blockLogs(ctx, header, false) + } + // Short-cut if all we care about is pending logs + if f.begin == rpc.PendingBlockNumber.Int64() { + if f.end != rpc.PendingBlockNumber.Int64() { + return nil, errors.New("invalid block range") + } + return f.pendingLogs() } // Figure out the limits of the filter range - header, _ := f.backend.HeaderByNumber(ctx, rpc.LatestBlockNumber) + header, _ := f.sys.backend.HeaderByNumber(ctx, rpc.LatestBlockNumber) if header == nil { return nil, nil } - head := header.Number.Uint64() - - if f.begin == -1 { + var ( + head = header.Number.Uint64() + end = uint64(f.end) + pending = f.end == rpc.PendingBlockNumber.Int64() + ) + if f.begin == rpc.LatestBlockNumber.Int64() { f.begin = int64(head) } - end := uint64(f.end) - if f.end == -1 { + if f.end == rpc.LatestBlockNumber.Int64() || f.end == rpc.PendingBlockNumber.Int64() { end = head } // Gather all indexed logs, and finish with non indexed ones var ( - logs []*types.Log - err error + logs []*types.Log + err error + size, sections = f.sys.backend.BloomStatus() ) - size, sections := f.backend.BloomStatus() if indexed := sections * size; indexed > uint64(f.begin) { if indexed > end { logs, err = f.indexedLogs(ctx, end) @@ -160,6 +147,13 @@ func (f *Filter) Logs(ctx context.Context) ([]*types.Log, error) { } rest, err := f.unindexedLogs(ctx, end) logs = append(logs, rest...) + if pending { + pendingLogs, err := f.pendingLogs() + if err != nil { + return nil, err + } + logs = append(logs, pendingLogs...) + } return logs, err } @@ -175,7 +169,7 @@ func (f *Filter) indexedLogs(ctx context.Context, end uint64) ([]*types.Log, err } defer session.Close() - f.backend.ServiceFilter(ctx, session) + f.sys.backend.ServiceFilter(ctx, session) // Iterate over the matches until exhausted or context closed var logs []*types.Log @@ -194,11 +188,11 @@ func (f *Filter) indexedLogs(ctx context.Context, end uint64) ([]*types.Log, err f.begin = int64(number) + 1 // Retrieve the suggested block and pull any truly matching logs - header, err := f.backend.HeaderByNumber(ctx, rpc.BlockNumber(number)) + header, err := f.sys.backend.HeaderByNumber(ctx, rpc.BlockNumber(number)) if header == nil || err != nil { return logs, err } - found, err := f.checkMatches(ctx, header) + found, err := f.blockLogs(ctx, header, true) if err != nil { return logs, err } @@ -216,11 +210,11 @@ func (f *Filter) unindexedLogs(ctx context.Context, end uint64) ([]*types.Log, e var logs []*types.Log for ; f.begin <= int64(end); f.begin++ { - header, err := f.backend.HeaderByNumber(ctx, rpc.BlockNumber(f.begin)) + header, err := f.sys.backend.HeaderByNumber(ctx, rpc.BlockNumber(f.begin)) if header == nil || err != nil { return logs, err } - found, err := f.blockLogs(ctx, header) + found, err := f.blockLogs(ctx, header, false) if err != nil { return logs, err } @@ -230,34 +224,34 @@ func (f *Filter) unindexedLogs(ctx context.Context, end uint64) ([]*types.Log, e } // blockLogs returns the logs matching the filter criteria within a single block. -func (f *Filter) blockLogs(ctx context.Context, header *types.Header) (logs []*types.Log, err error) { - if bloomFilter(header.Bloom, f.addresses, f.topics) { - found, err := f.checkMatches(ctx, header) +func (f *Filter) blockLogs(ctx context.Context, header *types.Header, skipBloom bool) ([]*types.Log, error) { + // Fast track: no filtering criteria + if len(f.addresses) == 0 && len(f.topics) == 0 { + list, err := f.sys.cachedGetLogs(ctx, header.Hash(), header.Number.Uint64()) if err != nil { - return logs, err + return nil, err } - logs = append(logs, found...) + return flatten(list), nil + } else if skipBloom || bloomFilter(header.Bloom, f.addresses, f.topics) { + return f.checkMatches(ctx, header) } - return logs, nil + return nil, nil } // checkMatches checks if the receipts belonging to the given header contain any log events that // match the filter criteria. This function is called when the bloom filter signals a potential match. -func (f *Filter) checkMatches(ctx context.Context, header *types.Header) (logs []*types.Log, err error) { - // Get the logs of the block - logsList, err := f.backend.GetLogs(ctx, header.Hash()) +func (f *Filter) checkMatches(ctx context.Context, header *types.Header) ([]*types.Log, error) { + logsList, err := f.sys.cachedGetLogs(ctx, header.Hash(), header.Number.Uint64()) if err != nil { return nil, err } - var unfiltered []*types.Log - for _, logs := range logsList { - unfiltered = append(unfiltered, logs...) - } - logs = filterLogs(unfiltered, nil, nil, f.addresses, f.topics) + + unfiltered := flatten(logsList) + logs := filterLogs(unfiltered, nil, nil, f.addresses, f.topics) if len(logs) > 0 { // We have matching logs, check if we need to resolve full logs via the light client if logs[0].TxHash == (common.Hash{}) { - receipts, err := f.backend.GetReceipts(ctx, header.Hash()) + receipts, err := f.sys.backend.GetReceipts(ctx, header.Hash()) if err != nil { return nil, err } @@ -272,6 +266,19 @@ func (f *Filter) checkMatches(ctx context.Context, header *types.Header) (logs [ return nil, nil } +// pendingLogs returns the logs matching the filter criteria within the pending block. +func (f *Filter) pendingLogs() ([]*types.Log, error) { + block, receipts := f.sys.backend.PendingBlockAndReceipts() + if bloomFilter(block.Bloom(), f.addresses, f.topics) { + var unfiltered []*types.Log + for _, r := range receipts { + unfiltered = append(unfiltered, r.Logs...) + } + return filterLogs(unfiltered, nil, nil, f.addresses, f.topics), nil + } + return nil, nil +} + func includes(addresses []common.Address, a common.Address) bool { for _, addr := range addresses { if addr == a { @@ -346,3 +353,11 @@ func bloomFilter(bloom types.Bloom, addresses []common.Address, topics [][]commo } return true } + +func flatten(list [][]*types.Log) []*types.Log { + var flat []*types.Log + for _, logs := range list { + flat = append(flat, logs...) + } + return flat +} diff --git a/eth/filters/filter_system.go b/eth/filters/filter_system.go index c1a1b408b7a74..79a9b089f4224 100644 --- a/eth/filters/filter_system.go +++ b/eth/filters/filter_system.go @@ -27,13 +27,90 @@ import ( "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/bloombits" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rpc" + lru "github.com/hashicorp/golang-lru" ) +// Config represents the configuration of the filter system. +type Config struct { + LogCacheSize int // maximum number of cached blocks (default: 32) + Timeout time.Duration // how long filters stay active (default: 5min) +} + +func (cfg Config) withDefaults() Config { + if cfg.Timeout == 0 { + cfg.Timeout = 5 * time.Minute + } + if cfg.LogCacheSize == 0 { + cfg.LogCacheSize = 32 + } + return cfg +} + +type Backend interface { + ChainDb() ethdb.Database + HeaderByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*types.Header, error) + HeaderByHash(ctx context.Context, blockHash common.Hash) (*types.Header, error) + GetReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error) + GetLogs(ctx context.Context, blockHash common.Hash, number uint64) ([][]*types.Log, error) + PendingBlockAndReceipts() (*types.Block, types.Receipts) + + SubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscription + SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription + SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription + SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription + SubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription + + BloomStatus() (uint64, uint64) + ServiceFilter(ctx context.Context, session *bloombits.MatcherSession) +} + +// FilterSystem holds resources shared by all filters. +type FilterSystem struct { + backend Backend + logsCache *lru.Cache + cfg *Config +} + +// NewFilterSystem creates a filter system. +func NewFilterSystem(backend Backend, config Config) *FilterSystem { + config = config.withDefaults() + + cache, err := lru.New(config.LogCacheSize) + if err != nil { + panic(err) + } + return &FilterSystem{ + backend: backend, + logsCache: cache, + cfg: &config, + } +} + +// cachedGetLogs loads block logs from the backend and caches the result. +func (sys *FilterSystem) cachedGetLogs(ctx context.Context, blockHash common.Hash, number uint64) ([][]*types.Log, error) { + cached, ok := sys.logsCache.Get(blockHash) + if ok { + return cached.([][]*types.Log), nil + } + + logs, err := sys.backend.GetLogs(ctx, blockHash, number) + if err != nil { + return nil, err + } + if logs == nil { + return nil, fmt.Errorf("failed to get logs for block #%d (0x%s)", number, blockHash.TerminalString()) + } + sys.logsCache.Add(blockHash, logs) + return logs, nil +} + // Type determines the kind of filter and is used to put the filter in to // the correct bucket when added. type Type byte @@ -84,6 +161,7 @@ type subscription struct { // subscription which match the subscription criteria. type EventSystem struct { backend Backend + sys *FilterSystem lightMode bool lastHead *types.Header @@ -110,9 +188,10 @@ type EventSystem struct { // // The returned manager has a loop that needs to be stopped with the Stop function // or by stopping the given mux. -func NewEventSystem(backend Backend, lightMode bool) *EventSystem { +func NewEventSystem(sys *FilterSystem, lightMode bool) *EventSystem { m := &EventSystem{ - backend: backend, + sys: sys, + backend: sys.backend, lightMode: lightMode, install: make(chan *subscription), uninstall: make(chan *subscription), @@ -405,7 +484,7 @@ func (es *EventSystem) lightFilterLogs(header *types.Header, addresses []common. // Get the logs of the block ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) defer cancel() - logsList, err := es.backend.GetLogs(ctx, header.Hash()) + logsList, err := es.sys.cachedGetLogs(ctx, header.Hash(), header.Number.Uint64()) if err != nil { return nil } diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go index 8d145d9604320..73a4ab2d4fca4 100644 --- a/eth/filters/filter_system_test.go +++ b/eth/filters/filter_system_test.go @@ -39,12 +39,7 @@ import ( "github.com/ethereum/go-ethereum/rpc" ) -var ( - deadline = 5 * time.Minute -) - type testBackend struct { - mux *event.TypeMux db ethdb.Database sections uint64 txFeed event.Feed @@ -92,20 +87,15 @@ func (b *testBackend) GetReceipts(ctx context.Context, hash common.Hash) (types. return nil, nil } -func (b *testBackend) GetLogs(ctx context.Context, hash common.Hash) ([][]*types.Log, error) { - number := rawdb.ReadHeaderNumber(b.db, hash) - if number == nil { - return nil, nil - } - receipts := rawdb.ReadReceipts(b.db, hash, *number, params.TestChainConfig) - - logs := make([][]*types.Log, len(receipts)) - for i, receipt := range receipts { - logs[i] = receipt.Logs - } +func (b *testBackend) GetLogs(ctx context.Context, hash common.Hash, number uint64) ([][]*types.Log, error) { + logs := rawdb.ReadLogs(b.db, hash, number, params.TestChainConfig) return logs, nil } +func (b *testBackend) PendingBlockAndReceipts() (*types.Block, types.Receipts) { + return nil, nil +} + func (b *testBackend) SubscribeNewTxsEvent(ch chan<- core.NewTxsEvent) event.Subscription { return b.txFeed.Subscribe(ch) } @@ -157,6 +147,12 @@ func (b *testBackend) ServiceFilter(ctx context.Context, session *bloombits.Matc }() } +func newTestFilterSystem(t testing.TB, db ethdb.Database, cfg Config) (*testBackend, *FilterSystem) { + backend := &testBackend{db: db} + sys := NewFilterSystem(backend, cfg) + return backend, sys +} + // TestBlockSubscription tests if a block subscription returns block hashes for posted chain events. // It creates multiple subscriptions: // - one at the start and should receive all posted chain events and a second (blockHashes) @@ -166,11 +162,14 @@ func TestBlockSubscription(t *testing.T) { t.Parallel() var ( - db = rawdb.NewMemoryDatabase() - backend = &testBackend{db: db} - api = NewPublicFilterAPI(backend, false, deadline) - genesis = (&core.Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(db) - chain, _ = core.GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), db, 10, func(i int, gen *core.BlockGen) {}) + db = rawdb.NewMemoryDatabase() + backend, sys = newTestFilterSystem(t, db, Config{}) + api = NewFilterAPI(sys, false) + genesis = &core.Genesis{ + Config: params.TestChainConfig, + BaseFee: big.NewInt(params.InitialBaseFee), + } + _, chain, _ = core.GenerateChainWithGenesis(genesis, ethash.NewFaker(), 10, func(i int, gen *core.BlockGen) {}) chainEvents = []core.ChainEvent{} ) @@ -218,9 +217,9 @@ func TestPendingTxFilter(t *testing.T) { t.Parallel() var ( - db = rawdb.NewMemoryDatabase() - backend = &testBackend{db: db} - api = NewPublicFilterAPI(backend, false, deadline) + db = rawdb.NewMemoryDatabase() + backend, sys = newTestFilterSystem(t, db, Config{}) + api = NewFilterAPI(sys, false) transactions = []*types.Transaction{ types.NewTransaction(0, common.HexToAddress("0xb794f5ea0ba39494ce83a213fffba74279579268"), new(big.Int), 0, new(big.Int), nil), @@ -273,9 +272,9 @@ func TestPendingTxFilter(t *testing.T) { // If not it must return an error. func TestLogFilterCreation(t *testing.T) { var ( - db = rawdb.NewMemoryDatabase() - backend = &testBackend{db: db} - api = NewPublicFilterAPI(backend, false, deadline) + db = rawdb.NewMemoryDatabase() + _, sys = newTestFilterSystem(t, db, Config{}) + api = NewFilterAPI(sys, false) testCases = []struct { crit FilterCriteria @@ -320,9 +319,9 @@ func TestInvalidLogFilterCreation(t *testing.T) { t.Parallel() var ( - db = rawdb.NewMemoryDatabase() - backend = &testBackend{db: db} - api = NewPublicFilterAPI(backend, false, deadline) + db = rawdb.NewMemoryDatabase() + _, sys = newTestFilterSystem(t, db, Config{}) + api = NewFilterAPI(sys, false) ) // different situations where log filter creation should fail. @@ -343,8 +342,8 @@ func TestInvalidLogFilterCreation(t *testing.T) { func TestInvalidGetLogsRequest(t *testing.T) { var ( db = rawdb.NewMemoryDatabase() - backend = &testBackend{db: db} - api = NewPublicFilterAPI(backend, false, deadline) + _, sys = newTestFilterSystem(t, db, Config{}) + api = NewFilterAPI(sys, false) blockHash = common.HexToHash("0x1111111111111111111111111111111111111111111111111111111111111111") ) @@ -367,9 +366,9 @@ func TestLogFilter(t *testing.T) { t.Parallel() var ( - db = rawdb.NewMemoryDatabase() - backend = &testBackend{db: db} - api = NewPublicFilterAPI(backend, false, deadline) + db = rawdb.NewMemoryDatabase() + backend, sys = newTestFilterSystem(t, db, Config{}) + api = NewFilterAPI(sys, false) firstAddr = common.HexToAddress("0x1111111111111111111111111111111111111111") secondAddr = common.HexToAddress("0x2222222222222222222222222222222222222222") @@ -481,9 +480,9 @@ func TestPendingLogsSubscription(t *testing.T) { t.Parallel() var ( - db = rawdb.NewMemoryDatabase() - backend = &testBackend{db: db} - api = NewPublicFilterAPI(backend, false, deadline) + db = rawdb.NewMemoryDatabase() + backend, sys = newTestFilterSystem(t, db, Config{}) + api = NewFilterAPI(sys, false) firstAddr = common.HexToAddress("0x1111111111111111111111111111111111111111") secondAddr = common.HexToAddress("0x2222222222222222222222222222222222222222") @@ -665,10 +664,10 @@ func TestPendingTxFilterDeadlock(t *testing.T) { timeout := 100 * time.Millisecond var ( - db = rawdb.NewMemoryDatabase() - backend = &testBackend{db: db} - api = NewPublicFilterAPI(backend, false, timeout) - done = make(chan struct{}) + db = rawdb.NewMemoryDatabase() + backend, sys = newTestFilterSystem(t, db, Config{Timeout: timeout}) + api = NewFilterAPI(sys, false) + done = make(chan struct{}) ) go func() { diff --git a/eth/filters/filter_test.go b/eth/filters/filter_test.go index f415046a82aab..39ed46cec761f 100644 --- a/eth/filters/filter_test.go +++ b/eth/filters/filter_test.go @@ -40,21 +40,23 @@ func makeReceipt(addr common.Address) *types.Receipt { } func BenchmarkFilters(b *testing.B) { - dir := b.TempDir() - var ( - db, _ = rawdb.NewLevelDBDatabase(dir, 0, 0, "", false) - backend = &testBackend{db: db} + db, _ = rawdb.NewLevelDBDatabase(b.TempDir(), 0, 0, "", false) + _, sys = newTestFilterSystem(b, db, Config{}) key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") addr1 = crypto.PubkeyToAddress(key1.PublicKey) addr2 = common.BytesToAddress([]byte("jeff")) addr3 = common.BytesToAddress([]byte("ethereum")) addr4 = common.BytesToAddress([]byte("random addresses please")) + + gspec = &core.Genesis{ + Alloc: core.GenesisAlloc{addr1: {Balance: big.NewInt(1000000)}}, + BaseFee: big.NewInt(params.InitialBaseFee), + } ) defer db.Close() - genesis := core.GenesisBlockForTesting(db, addr1, big.NewInt(1000000)) - chain, receipts := core.GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), db, 100010, func(i int, gen *core.BlockGen) { + _, chain, receipts := core.GenerateChainWithGenesis(gspec, ethash.NewFaker(), 100010, func(i int, gen *core.BlockGen) { switch i { case 2403: receipt := makeReceipt(addr1) @@ -72,7 +74,6 @@ func BenchmarkFilters(b *testing.B) { receipt := makeReceipt(addr4) gen.AddUncheckedReceipt(receipt) gen.AddUncheckedTx(types.NewTransaction(999, common.HexToAddress("0x999"), big.NewInt(999), 999, gen.BaseFee(), nil)) - } }) for i, block := range chain { @@ -83,7 +84,7 @@ func BenchmarkFilters(b *testing.B) { } b.ResetTimer() - filter := NewRangeFilter(backend, 0, -1, []common.Address{addr1, addr2, addr3, addr4}, nil) + filter := sys.NewRangeFilter(0, -1, []common.Address{addr1, addr2, addr3, addr4}, nil) for i := 0; i < b.N; i++ { logs, _ := filter.Logs(context.Background()) @@ -94,11 +95,9 @@ func BenchmarkFilters(b *testing.B) { } func TestFilters(t *testing.T) { - dir := t.TempDir() - var ( - db, _ = rawdb.NewLevelDBDatabase(dir, 0, 0, "", false) - backend = &testBackend{db: db} + db, _ = rawdb.NewLevelDBDatabase(t.TempDir(), 0, 0, "", false) + _, sys = newTestFilterSystem(t, db, Config{}) key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") addr = crypto.PubkeyToAddress(key1.PublicKey) @@ -106,11 +105,16 @@ func TestFilters(t *testing.T) { hash2 = common.BytesToHash([]byte("topic2")) hash3 = common.BytesToHash([]byte("topic3")) hash4 = common.BytesToHash([]byte("topic4")) + + gspec = &core.Genesis{ + Config: params.TestChainConfig, + Alloc: core.GenesisAlloc{addr: {Balance: big.NewInt(1000000)}}, + BaseFee: big.NewInt(params.InitialBaseFee), + } ) defer db.Close() - genesis := core.GenesisBlockForTesting(db, addr, big.NewInt(1000000)) - chain, receipts := core.GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), db, 1000, func(i int, gen *core.BlockGen) { + _, chain, receipts := core.GenerateChainWithGenesis(gspec, ethash.NewFaker(), 1000, func(i int, gen *core.BlockGen) { switch i { case 1: receipt := types.NewReceipt(nil, false, 0) @@ -155,6 +159,10 @@ func TestFilters(t *testing.T) { gen.AddUncheckedTx(types.NewTransaction(999, common.HexToAddress("0x999"), big.NewInt(999), 999, gen.BaseFee(), nil)) } }) + // The test txs are not properly signed, can't simply create a chain + // and then import blocks. TODO(rjl493456442) try to get rid of the + // manual database writes. + gspec.MustCommit(db) for i, block := range chain { rawdb.WriteBlock(db, block) rawdb.WriteCanonicalHash(db, block.Hash(), block.NumberU64()) @@ -162,14 +170,14 @@ func TestFilters(t *testing.T) { rawdb.WriteReceipts(db, block.Hash(), block.NumberU64(), receipts[i]) } - filter := NewRangeFilter(backend, 0, -1, []common.Address{addr}, [][]common.Hash{{hash1, hash2, hash3, hash4}}) + filter := sys.NewRangeFilter(0, -1, []common.Address{addr}, [][]common.Hash{{hash1, hash2, hash3, hash4}}) logs, _ := filter.Logs(context.Background()) if len(logs) != 4 { t.Error("expected 4 log, got", len(logs)) } - filter = NewRangeFilter(backend, 900, 999, []common.Address{addr}, [][]common.Hash{{hash3}}) + filter = sys.NewRangeFilter(900, 999, []common.Address{addr}, [][]common.Hash{{hash3}}) logs, _ = filter.Logs(context.Background()) if len(logs) != 1 { t.Error("expected 1 log, got", len(logs)) @@ -178,7 +186,7 @@ func TestFilters(t *testing.T) { t.Errorf("expected log[0].Topics[0] to be %x, got %x", hash3, logs[0].Topics[0]) } - filter = NewRangeFilter(backend, 990, -1, []common.Address{addr}, [][]common.Hash{{hash3}}) + filter = sys.NewRangeFilter(990, -1, []common.Address{addr}, [][]common.Hash{{hash3}}) logs, _ = filter.Logs(context.Background()) if len(logs) != 1 { t.Error("expected 1 log, got", len(logs)) @@ -187,7 +195,7 @@ func TestFilters(t *testing.T) { t.Errorf("expected log[0].Topics[0] to be %x, got %x", hash3, logs[0].Topics[0]) } - filter = NewRangeFilter(backend, 1, 10, nil, [][]common.Hash{{hash1, hash2}}) + filter = sys.NewRangeFilter(1, 10, nil, [][]common.Hash{{hash1, hash2}}) logs, _ = filter.Logs(context.Background()) if len(logs) != 2 { @@ -195,7 +203,7 @@ func TestFilters(t *testing.T) { } failHash := common.BytesToHash([]byte("fail")) - filter = NewRangeFilter(backend, 0, -1, nil, [][]common.Hash{{failHash}}) + filter = sys.NewRangeFilter(0, -1, nil, [][]common.Hash{{failHash}}) logs, _ = filter.Logs(context.Background()) if len(logs) != 0 { @@ -203,14 +211,14 @@ func TestFilters(t *testing.T) { } failAddr := common.BytesToAddress([]byte("failmenow")) - filter = NewRangeFilter(backend, 0, -1, []common.Address{failAddr}, nil) + filter = sys.NewRangeFilter(0, -1, []common.Address{failAddr}, nil) logs, _ = filter.Logs(context.Background()) if len(logs) != 0 { t.Error("expected 0 log, got", len(logs)) } - filter = NewRangeFilter(backend, 0, -1, nil, [][]common.Hash{{failHash}, {hash1}}) + filter = sys.NewRangeFilter(0, -1, nil, [][]common.Hash{{failHash}, {hash1}}) logs, _ = filter.Logs(context.Background()) if len(logs) != 0 { diff --git a/eth/gasprice/feehistory.go b/eth/gasprice/feehistory.go index 4113089afb1eb..6028ef03cf154 100644 --- a/eth/gasprice/feehistory.go +++ b/eth/gasprice/feehistory.go @@ -137,44 +137,68 @@ func (oracle *Oracle) processBlock(bf *blockFees, percentiles []float64) { // also returned if requested and available. // Note: an error is only returned if retrieving the head header has failed. If there are no // retrievable blocks in the specified range then zero block count is returned with no error. -func (oracle *Oracle) resolveBlockRange(ctx context.Context, lastBlock rpc.BlockNumber, blocks int) (*types.Block, []*types.Receipt, uint64, int, error) { +func (oracle *Oracle) resolveBlockRange(ctx context.Context, reqEnd rpc.BlockNumber, blocks int) (*types.Block, []*types.Receipt, uint64, int, error) { var ( - headBlock rpc.BlockNumber + headBlock *types.Header pendingBlock *types.Block pendingReceipts types.Receipts + err error ) - // query either pending block or head header and set headBlock - if lastBlock == rpc.PendingBlockNumber { - if pendingBlock, pendingReceipts = oracle.backend.PendingBlockAndReceipts(); pendingBlock != nil { - lastBlock = rpc.BlockNumber(pendingBlock.NumberU64()) - headBlock = lastBlock - 1 - } else { - // pending block not supported by backend, process until latest block - lastBlock = rpc.LatestBlockNumber - blocks-- - if blocks == 0 { - return nil, nil, 0, 0, nil + + // Get the chain's current head. + if headBlock, err = oracle.backend.HeaderByNumber(ctx, rpc.LatestBlockNumber); err != nil { + return nil, nil, 0, 0, err + } + head := rpc.BlockNumber(headBlock.Number.Uint64()) + + // Fail if request block is beyond the chain's current head. + if head < reqEnd { + return nil, nil, 0, 0, fmt.Errorf("%w: requested %d, head %d", errRequestBeyondHead, reqEnd, head) + } + + // Resolve block tag. + if reqEnd < 0 { + var ( + resolved *types.Header + err error + ) + switch reqEnd { + case rpc.PendingBlockNumber: + if pendingBlock, pendingReceipts = oracle.backend.PendingBlockAndReceipts(); pendingBlock != nil { + resolved = pendingBlock.Header() + } else { + // Pending block not supported by backend, process only until latest block. + resolved = headBlock + + // Update total blocks to return to account for this. + blocks-- } + case rpc.LatestBlockNumber: + // Retrieved above. + resolved = headBlock + case rpc.SafeBlockNumber: + resolved, err = oracle.backend.HeaderByNumber(ctx, rpc.SafeBlockNumber) + case rpc.FinalizedBlockNumber: + resolved, err = oracle.backend.HeaderByNumber(ctx, rpc.FinalizedBlockNumber) + case rpc.EarliestBlockNumber: + resolved, err = oracle.backend.HeaderByNumber(ctx, rpc.EarliestBlockNumber) } - } - if pendingBlock == nil { - // if pending block is not fetched then we retrieve the head header to get the head block number - if latestHeader, err := oracle.backend.HeaderByNumber(ctx, rpc.LatestBlockNumber); err == nil { - headBlock = rpc.BlockNumber(latestHeader.Number.Uint64()) - } else { + if resolved == nil || err != nil { return nil, nil, 0, 0, err } + // Absolute number resolved. + reqEnd = rpc.BlockNumber(resolved.Number.Uint64()) } - if lastBlock == rpc.LatestBlockNumber { - lastBlock = headBlock - } else if pendingBlock == nil && lastBlock > headBlock { - return nil, nil, 0, 0, fmt.Errorf("%w: requested %d, head %d", errRequestBeyondHead, lastBlock, headBlock) + + // If there are no blocks to return, short circuit. + if blocks == 0 { + return nil, nil, 0, 0, nil } - // ensure not trying to retrieve before genesis - if rpc.BlockNumber(blocks) > lastBlock+1 { - blocks = int(lastBlock + 1) + // Ensure not trying to retrieve before genesis. + if int(reqEnd+1) < blocks { + blocks = int(reqEnd + 1) } - return pendingBlock, pendingReceipts, uint64(lastBlock), blocks, nil + return pendingBlock, pendingReceipts, uint64(reqEnd), blocks, nil } // FeeHistory returns data relevant for fee estimation based on the specified range of blocks. @@ -184,10 +208,11 @@ func (oracle *Oracle) resolveBlockRange(ctx context.Context, lastBlock rpc.Block // actually processed range is returned to avoid ambiguity when parts of the requested range // are not available or when the head has changed during processing this request. // Three arrays are returned based on the processed blocks: -// - reward: the requested percentiles of effective priority fees per gas of transactions in each -// block, sorted in ascending order and weighted by gas used. -// - baseFee: base fee per gas in the given block -// - gasUsedRatio: gasUsed/gasLimit in the given block +// - reward: the requested percentiles of effective priority fees per gas of transactions in each +// block, sorted in ascending order and weighted by gas used. +// - baseFee: base fee per gas in the given block +// - gasUsedRatio: gasUsed/gasLimit in the given block +// // Note: baseFee includes the next block after the newest of the returned range, because this // value can be derived from the newest block. func (oracle *Oracle) FeeHistory(ctx context.Context, blocks int, unresolvedLastBlock rpc.BlockNumber, rewardPercentiles []float64) (*big.Int, [][]*big.Int, []*big.Int, []float64, error) { diff --git a/eth/gasprice/feehistory_test.go b/eth/gasprice/feehistory_test.go index c259eb0acf762..deece7f461500 100644 --- a/eth/gasprice/feehistory_test.go +++ b/eth/gasprice/feehistory_test.go @@ -50,6 +50,8 @@ func TestFeeHistory(t *testing.T) { {false, 1000, 1000, 2, rpc.PendingBlockNumber, nil, 32, 1, nil}, {true, 1000, 1000, 2, rpc.PendingBlockNumber, nil, 32, 2, nil}, {true, 1000, 1000, 2, rpc.PendingBlockNumber, []float64{0, 10}, 32, 2, nil}, + {false, 1000, 1000, 2, rpc.FinalizedBlockNumber, []float64{0, 10}, 24, 2, nil}, + {false, 1000, 1000, 2, rpc.SafeBlockNumber, []float64{0, 10}, 24, 2, nil}, } for i, c := range cases { config := Config{ diff --git a/eth/gasprice/gasprice_test.go b/eth/gasprice/gasprice_test.go index c0d3c6b6038e9..a4399661fcf04 100644 --- a/eth/gasprice/gasprice_test.go +++ b/eth/gasprice/gasprice_test.go @@ -45,6 +45,15 @@ func (b *testBackend) HeaderByNumber(ctx context.Context, number rpc.BlockNumber if number > testHead { return nil, nil } + if number == rpc.EarliestBlockNumber { + number = 0 + } + if number == rpc.FinalizedBlockNumber { + return b.chain.CurrentFinalizedBlock().Header(), nil + } + if number == rpc.SafeBlockNumber { + return b.chain.CurrentSafeBlock().Header(), nil + } if number == rpc.LatestBlockNumber { number = testHead } @@ -62,6 +71,15 @@ func (b *testBackend) BlockByNumber(ctx context.Context, number rpc.BlockNumber) if number > testHead { return nil, nil } + if number == rpc.EarliestBlockNumber { + number = 0 + } + if number == rpc.FinalizedBlockNumber { + return b.chain.CurrentFinalizedBlock(), nil + } + if number == rpc.SafeBlockNumber { + return b.chain.CurrentSafeBlock(), nil + } if number == rpc.LatestBlockNumber { number = testHead } @@ -108,14 +126,12 @@ func newTestBackend(t *testing.T, londonBlock *big.Int, pending bool) *testBacke ) config.LondonBlock = londonBlock config.ArrowGlacierBlock = londonBlock + config.GrayGlacierBlock = londonBlock + config.TerminalTotalDifficulty = common.Big0 engine := ethash.NewFaker() - db := rawdb.NewMemoryDatabase() - genesis, err := gspec.Commit(db) - if err != nil { - t.Fatal(err) - } + // Generate testing blocks - blocks, _ := core.GenerateChain(gspec.Config, genesis, engine, db, testHead+1, func(i int, b *core.BlockGen) { + _, blocks, _ := core.GenerateChainWithGenesis(gspec, engine, testHead+1, func(i int, b *core.BlockGen) { b.SetCoinbase(common.Address{1}) var txdata types.TxData @@ -142,13 +158,13 @@ func newTestBackend(t *testing.T, londonBlock *big.Int, pending bool) *testBacke b.AddTx(types.MustSignNewTx(key, signer, txdata)) }) // Construct testing chain - diskdb := rawdb.NewMemoryDatabase() - gspec.Commit(diskdb) - chain, err := core.NewBlockChain(diskdb, &core.CacheConfig{TrieCleanNoPrefetch: true}, gspec.Config, engine, vm.Config{}, nil, nil) + chain, err := core.NewBlockChain(rawdb.NewMemoryDatabase(), &core.CacheConfig{TrieCleanNoPrefetch: true}, gspec, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to create local chain, %v", err) } chain.InsertChain(blocks) + chain.SetFinalized(chain.GetBlockByNumber(25)) + chain.SetSafe(chain.GetBlockByNumber(25)) return &testBackend{chain: chain, pending: pending} } diff --git a/eth/handler.go b/eth/handler.go index 54efe18d64a1b..143147b0c8153 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -191,11 +191,22 @@ func newHandler(config *handlerConfig) (*handler, error) { } } } - // Construct the downloader (long sync) and its backing state bloom if snap - // sync is requested. The downloader is responsible for deallocating the state - // bloom when it's done. + // Construct the downloader (long sync) h.downloader = downloader.New(h.checkpointNumber, config.Database, h.eventMux, h.chain, nil, h.removePeer, success) - + if ttd := h.chain.Config().TerminalTotalDifficulty; ttd != nil { + if h.chain.Config().TerminalTotalDifficultyPassed { + log.Info("Chain post-merge, sync via beacon client") + } else { + head := h.chain.CurrentBlock() + if td := h.chain.GetTd(head.Hash(), head.NumberU64()); td.Cmp(ttd) >= 0 { + log.Info("Chain post-TTD, sync via beacon client") + } else { + log.Warn("Chain pre-merge, sync via PoW (ensure beacon client is ready)") + } + } + } else if h.chain.Config().TerminalTotalDifficultyPassed { + log.Error("Chain configured post-merge, but without TTD. Are you debugging sync?") + } // Construct the fetcher (short sync) validator := func(header *types.Header) error { // All the block fetcher activities should be disabled @@ -249,7 +260,7 @@ func newHandler(config *handlerConfig) (*handler, error) { // out a way yet where nodes can decide unilaterally whether the network is new // or not. This should be fixed if we figure out a solution. if atomic.LoadUint32(&h.snapSync) == 1 { - log.Warn("Fast syncing, discarded propagated block", "number", blocks[0].Number(), "hash", blocks[0].Hash()) + log.Warn("Snap syncing, discarded propagated block", "number", blocks[0].Number(), "hash", blocks[0].Hash()) return 0, nil } if h.merger.TDDReached() { @@ -296,7 +307,7 @@ func newHandler(config *handlerConfig) (*handler, error) { } // runEthPeer registers an eth peer into the joint eth/snap peerset, adds it to -// various subsistems and starts handling messages. +// various subsystems and starts handling messages. func (h *handler) runEthPeer(peer *eth.Peer, handler eth.Handler) error { // If the peer has a `snap` extension, wait for it to connect so we can have // a uniform initialization/teardown mechanism @@ -380,11 +391,16 @@ func (h *handler) runEthPeer(peer *eth.Peer, handler eth.Handler) error { if h.checkpointHash != (common.Hash{}) { // Request the peer's checkpoint header for chain height/weight validation resCh := make(chan *eth.Response) - if _, err := peer.RequestHeadersByNumber(h.checkpointNumber, 1, 0, false, resCh); err != nil { + + req, err := peer.RequestHeadersByNumber(h.checkpointNumber, 1, 0, false, resCh) + if err != nil { return err } // Start a timer to disconnect if the peer doesn't reply in time go func() { + // Ensure the request gets cancelled in case of error/drop + defer req.Close() + timeout := time.NewTimer(syncChallengeTimeout) defer timeout.Stop() @@ -426,10 +442,15 @@ func (h *handler) runEthPeer(peer *eth.Peer, handler eth.Handler) error { // If we have any explicit peer required block hashes, request them for number, hash := range h.requiredBlocks { resCh := make(chan *eth.Response) - if _, err := peer.RequestHeadersByNumber(number, 1, 0, false, resCh); err != nil { + + req, err := peer.RequestHeadersByNumber(number, 1, 0, false, resCh) + if err != nil { return err } - go func(number uint64, hash common.Hash) { + go func(number uint64, hash common.Hash, req *eth.Request) { + // Ensure the request gets cancelled in case of error/drop + defer req.Close() + timeout := time.NewTimer(syncChallengeTimeout) defer timeout.Stop() @@ -458,7 +479,7 @@ func (h *handler) runEthPeer(peer *eth.Peer, handler eth.Handler) error { peer.Log().Warn("Required block challenge timed out, dropping", "addr", peer.RemoteAddr(), "type", peer.Name()) h.removePeer(peer.ID()) } - }(number, hash) + }(number, hash, req) } // Handle incoming messages until the connection is torn down return handler(peer) diff --git a/eth/handler_eth_test.go b/eth/handler_eth_test.go index dffbfbe612a2b..9f0c36f950c5a 100644 --- a/eth/handler_eth_test.go +++ b/eth/handler_eth_test.go @@ -102,14 +102,11 @@ func testForkIDSplit(t *testing.T, protocol uint) { gspecNoFork = &core.Genesis{Config: configNoFork} gspecProFork = &core.Genesis{Config: configProFork} - genesisNoFork = gspecNoFork.MustCommit(dbNoFork) - genesisProFork = gspecProFork.MustCommit(dbProFork) + chainNoFork, _ = core.NewBlockChain(dbNoFork, nil, gspecNoFork, nil, engine, vm.Config{}, nil, nil) + chainProFork, _ = core.NewBlockChain(dbProFork, nil, gspecProFork, nil, engine, vm.Config{}, nil, nil) - chainNoFork, _ = core.NewBlockChain(dbNoFork, nil, configNoFork, engine, vm.Config{}, nil, nil) - chainProFork, _ = core.NewBlockChain(dbProFork, nil, configProFork, engine, vm.Config{}, nil, nil) - - blocksNoFork, _ = core.GenerateChain(configNoFork, genesisNoFork, engine, dbNoFork, 2, nil) - blocksProFork, _ = core.GenerateChain(configProFork, genesisProFork, engine, dbProFork, 2, nil) + _, blocksNoFork, _ = core.GenerateChainWithGenesis(gspecNoFork, engine, 2, nil) + _, blocksProFork, _ = core.GenerateChainWithGenesis(gspecProFork, engine, 2, nil) ethNoFork, _ = newHandler(&handlerConfig{ Database: dbNoFork, @@ -490,7 +487,6 @@ func TestCheckpointChallenge(t *testing.T) { } func testCheckpointChallenge(t *testing.T, syncmode downloader.SyncMode, checkpoint bool, timeout bool, empty bool, match bool, drop bool) { - // Reduce the checkpoint handshake challenge timeout defer func(old time.Duration) { syncChallengeTimeout = old }(syncChallengeTimeout) syncChallengeTimeout = 250 * time.Millisecond diff --git a/eth/handler_test.go b/eth/handler_test.go index d967b6df935ed..8939e53a952a8 100644 --- a/eth/handler_test.go +++ b/eth/handler_test.go @@ -133,14 +133,13 @@ func newTestHandler() *testHandler { func newTestHandlerWithBlocks(blocks int) *testHandler { // Create a database pre-initialize with a genesis block db := rawdb.NewMemoryDatabase() - (&core.Genesis{ + gspec := &core.Genesis{ Config: params.TestChainConfig, Alloc: core.GenesisAlloc{testAddr: {Balance: big.NewInt(1000000)}}, - }).MustCommit(db) - - chain, _ := core.NewBlockChain(db, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil) + } + chain, _ := core.NewBlockChain(db, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) - bs, _ := core.GenerateChain(params.TestChainConfig, chain.Genesis(), ethash.NewFaker(), db, blocks, nil) + _, bs, _ := core.GenerateChainWithGenesis(gspec, ethash.NewFaker(), blocks, nil) if _, err := chain.InsertChain(bs); err != nil { panic(err) } diff --git a/eth/peer.go b/eth/peer.go index 024a6e619371f..55e5f0046206a 100644 --- a/eth/peer.go +++ b/eth/peer.go @@ -34,8 +34,7 @@ type ethPeerInfo struct { // ethPeer is a wrapper around eth.Peer to maintain a few extra metadata. type ethPeer struct { *eth.Peer - snapExt *snapPeer // Satellite `snap` connection - snapWait chan struct{} // Notification channel for snap connections + snapExt *snapPeer // Satellite `snap` connection } // info gathers and returns some `eth` protocol metadata known about a peer. diff --git a/eth/peerset.go b/eth/peerset.go index 3e54a481e36b4..b9cc1e03aca3d 100644 --- a/eth/peerset.go +++ b/eth/peerset.go @@ -41,7 +41,7 @@ var ( errPeerNotRegistered = errors.New("peer not registered") // errSnapWithoutEth is returned if a peer attempts to connect only on the - // snap protocol without advertizing the eth main protocol. + // snap protocol without advertising the eth main protocol. errSnapWithoutEth = errors.New("peer connected on snap without compatible eth support") ) diff --git a/eth/protocols/eth/broadcast.go b/eth/protocols/eth/broadcast.go index 09330cfdf320a..6fc15f136bff9 100644 --- a/eth/protocols/eth/broadcast.go +++ b/eth/protocols/eth/broadcast.go @@ -36,7 +36,7 @@ type blockPropagation struct { td *big.Int } -// broadcastBlocks is a write loop that multiplexes blocks and block accouncements +// broadcastBlocks is a write loop that multiplexes blocks and block announcements // to the remote peer. The goal is to have an async writer that does not lock up // node internals and at the same time rate limits queued data. func (p *Peer) broadcastBlocks() { diff --git a/eth/protocols/eth/dispatcher.go b/eth/protocols/eth/dispatcher.go index bf88d400d4a0c..65a935d555482 100644 --- a/eth/protocols/eth/dispatcher.go +++ b/eth/protocols/eth/dispatcher.go @@ -224,7 +224,7 @@ func (p *Peer) dispatcher() { switch { case res.Req == nil: // Response arrived with an untracked ID. Since even cancelled - // requests are tracked until fulfilment, a dangling repsponse + // requests are tracked until fulfilment, a dangling response // means the remote peer implements the protocol badly. resOp.fail <- errDanglingResponse diff --git a/eth/protocols/eth/handler.go b/eth/protocols/eth/handler.go index 81d45d8b8fcf0..87b1f20a2dc2f 100644 --- a/eth/protocols/eth/handler.go +++ b/eth/protocols/eth/handler.go @@ -127,7 +127,7 @@ func MakeProtocols(backend Backend, network uint64, dnsdisc enode.Iterator) []p2 // NodeInfo represents a short summary of the `eth` sub-protocol metadata // known about the host peer. type NodeInfo struct { - Network uint64 `json:"network"` // Ethereum network ID (1=Frontier, 2=Morden, Ropsten=3, Rinkeby=4) + Network uint64 `json:"network"` // Ethereum network ID (1=Mainnet, Ropsten=3, Rinkeby=4, Goerli=5) Difficulty *big.Int `json:"difficulty"` // Total difficulty of the host's blockchain Genesis common.Hash `json:"genesis"` // SHA3 hash of the host's genesis block Config *params.ChainConfig `json:"config"` // Chain configuration for the fork rules @@ -181,6 +181,21 @@ var eth66 = map[uint64]msgHandler{ PooledTransactionsMsg: handlePooledTransactions66, } +var eth67 = map[uint64]msgHandler{ + NewBlockHashesMsg: handleNewBlockhashes, + NewBlockMsg: handleNewBlock, + TransactionsMsg: handleTransactions, + NewPooledTransactionHashesMsg: handleNewPooledTransactionHashes, + GetBlockHeadersMsg: handleGetBlockHeaders66, + BlockHeadersMsg: handleBlockHeaders66, + GetBlockBodiesMsg: handleGetBlockBodies66, + BlockBodiesMsg: handleBlockBodies66, + GetReceiptsMsg: handleGetReceipts66, + ReceiptsMsg: handleReceipts66, + GetPooledTransactionsMsg: handleGetPooledTransactions66, + PooledTransactionsMsg: handlePooledTransactions66, +} + // handleMessage is invoked whenever an inbound message is received from a remote // peer. The remote connection is torn down upon returning any error. func handleMessage(backend Backend, peer *Peer) error { @@ -195,9 +210,9 @@ func handleMessage(backend Backend, peer *Peer) error { defer msg.Discard() var handlers = eth66 - //if peer.Version() >= ETH67 { // Left in as a sample when new protocol is added - // handlers = eth67 - //} + if peer.Version() >= ETH67 { + handlers = eth67 + } // Track the amount of time it takes to serve the request and run the handler if metrics.Enabled { diff --git a/eth/protocols/eth/handler_test.go b/eth/protocols/eth/handler_test.go index bf836e8f51323..ef534ba376977 100644 --- a/eth/protocols/eth/handler_test.go +++ b/eth/protocols/eth/handler_test.go @@ -63,17 +63,19 @@ func newTestBackend(blocks int) *testBackend { func newTestBackendWithGenerator(blocks int, generator func(int, *core.BlockGen)) *testBackend { // Create a database pre-initialize with a genesis block db := rawdb.NewMemoryDatabase() - (&core.Genesis{ + gspec := &core.Genesis{ Config: params.TestChainConfig, Alloc: core.GenesisAlloc{testAddr: {Balance: big.NewInt(100_000_000_000_000_000)}}, - }).MustCommit(db) - - chain, _ := core.NewBlockChain(db, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil) + } + chain, _ := core.NewBlockChain(db, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) - bs, _ := core.GenerateChain(params.TestChainConfig, chain.Genesis(), ethash.NewFaker(), db, blocks, generator) + _, bs, _ := core.GenerateChainWithGenesis(gspec, ethash.NewFaker(), blocks, generator) if _, err := chain.InsertChain(bs); err != nil { panic(err) } + for _, block := range bs { + chain.StateCache().TrieDB().Commit(block.Root(), false, nil) + } txconfig := core.DefaultTxPoolConfig txconfig.Journal = "" // Don't litter the disk with test journals @@ -94,7 +96,7 @@ func (b *testBackend) Chain() *core.BlockChain { return b.chain } func (b *testBackend) TxPool() TxPool { return b.txpool } func (b *testBackend) RunPeer(peer *Peer, handler Handler) error { - // Normally the backend would do peer mainentance and handshakes. All that + // Normally the backend would do peer maintenance and handshakes. All that // is omitted and we will just give control back to the handler. return handler(peer) } diff --git a/eth/protocols/eth/peer.go b/eth/protocols/eth/peer.go index 22674d65b041f..a23726384d70a 100644 --- a/eth/protocols/eth/peer.go +++ b/eth/protocols/eth/peer.go @@ -133,7 +133,7 @@ func (p *Peer) ID() string { return p.id } -// Version retrieves the peer's negoatiated `eth` protocol version. +// Version retrieves the peer's negotiated `eth` protocol version. func (p *Peer) Version() uint { return p.version } diff --git a/eth/protocols/eth/protocol.go b/eth/protocols/eth/protocol.go index 24b65f01dd96a..f6fac42780805 100644 --- a/eth/protocols/eth/protocol.go +++ b/eth/protocols/eth/protocol.go @@ -31,6 +31,7 @@ import ( // Constants to match up protocol versions and messages const ( ETH66 = 66 + ETH67 = 67 ) // ProtocolName is the official short name of the `eth` protocol used during @@ -39,11 +40,11 @@ const ProtocolName = "eth" // ProtocolVersions are the supported versions of the `eth` protocol (first // is primary). -var ProtocolVersions = []uint{ETH66} +var ProtocolVersions = []uint{ETH67, ETH66} // protocolLengths are the number of implemented message corresponding to // different protocol versions. -var protocolLengths = map[uint]uint64{ETH66: 17} +var protocolLengths = map[uint]uint64{ETH67: 17, ETH66: 17} // maxMessageSize is the maximum cap on the size of a protocol message. const maxMessageSize = 10 * 1024 * 1024 diff --git a/eth/protocols/eth/protocol_test.go b/eth/protocols/eth/protocol_test.go index 5ca8957741210..a86fbb0a6906a 100644 --- a/eth/protocols/eth/protocol_test.go +++ b/eth/protocols/eth/protocol_test.go @@ -115,12 +115,10 @@ func TestEth66EmptyMessages(t *testing.T) { t.Errorf("test %d, type %T, have\n\t%x\nwant\n\t%x", i, msg, have, want) } } - } // TestEth66Messages tests the encoding of all redefined eth66 messages func TestEth66Messages(t *testing.T) { - // Some basic structs used during testing var ( header *types.Header diff --git a/eth/protocols/snap/handler.go b/eth/protocols/snap/handler.go index 23638ef888849..aa245ab7e62d0 100644 --- a/eth/protocols/snap/handler.go +++ b/eth/protocols/snap/handler.go @@ -23,14 +23,12 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/light" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/p2p/enr" - "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" ) @@ -285,7 +283,7 @@ func ServiceGetAccountRangeQuery(chain *core.BlockChain, req *GetAccountRangePac req.Bytes = softResponseLimit } // Retrieve the requested state and bail out if non existent - tr, err := trie.New(req.Root, chain.StateCache().TrieDB()) + tr, err := trie.New(trie.StateTrieID(req.Root), chain.StateCache().TrieDB()) if err != nil { return nil, nil } @@ -415,15 +413,16 @@ func ServiceGetStorageRangesQuery(chain *core.BlockChain, req *GetStorageRangesP if origin != (common.Hash{}) || (abort && len(storage) > 0) { // Request started at a non-zero hash or was capped prematurely, add // the endpoint Merkle proofs - accTrie, err := trie.New(req.Root, chain.StateCache().TrieDB()) + accTrie, err := trie.NewStateTrie(trie.StateTrieID(req.Root), chain.StateCache().TrieDB()) if err != nil { return nil, nil } - var acc types.StateAccount - if err := rlp.DecodeBytes(accTrie.Get(account[:]), &acc); err != nil { + acc, err := accTrie.TryGetAccountWithPreHashedKey(account[:]) + if err != nil || acc == nil { return nil, nil } - stTrie, err := trie.New(acc.Root, chain.StateCache().TrieDB()) + id := trie.StorageTrieID(req.Root, account, acc.Root) + stTrie, err := trie.NewStateTrie(id, chain.StateCache().TrieDB()) if err != nil { return nil, nil } @@ -489,7 +488,7 @@ func ServiceGetTrieNodesQuery(chain *core.BlockChain, req *GetTrieNodesPacket, s // Make sure we have the state associated with the request triedb := chain.StateCache().TrieDB() - accTrie, err := trie.NewSecure(req.Root, triedb) + accTrie, err := trie.NewStateTrie(trie.StateTrieID(req.Root), triedb) if err != nil { // We don't have the requested state available, bail out return nil, nil @@ -506,7 +505,7 @@ func ServiceGetTrieNodesQuery(chain *core.BlockChain, req *GetTrieNodesPacket, s var ( nodes [][]byte bytes uint64 - loads int // Trie hash expansions to cound database reads + loads int // Trie hash expansions to count database reads ) for _, pathset := range req.Paths { switch len(pathset) { @@ -531,7 +530,8 @@ func ServiceGetTrieNodesQuery(chain *core.BlockChain, req *GetTrieNodesPacket, s if err != nil || account == nil { break } - stTrie, err := trie.NewSecure(common.BytesToHash(account.Root), triedb) + id := trie.StorageTrieID(req.Root, common.BytesToHash(pathset[0]), common.BytesToHash(account.Root)) + stTrie, err := trie.NewStateTrie(id, triedb) loads++ // always account database reads, even for failures if err != nil { break diff --git a/eth/protocols/snap/peer.go b/eth/protocols/snap/peer.go index 87a62d2f8a413..235d499ffdc98 100644 --- a/eth/protocols/snap/peer.go +++ b/eth/protocols/snap/peer.go @@ -61,12 +61,12 @@ func (p *Peer) ID() string { return p.id } -// Version retrieves the peer's negoatiated `snap` protocol version. +// Version retrieves the peer's negotiated `snap` protocol version. func (p *Peer) Version() uint { return p.version } -// Log overrides the P2P logget with the higher level one containing only the id. +// Log overrides the P2P logger with the higher level one containing only the id. func (p *Peer) Log() log.Logger { return p.logger } @@ -87,7 +87,7 @@ func (p *Peer) RequestAccountRange(id uint64, root common.Hash, origin, limit co } // RequestStorageRange fetches a batch of storage slots belonging to one or more -// accounts. If slots from only one accout is requested, an origin marker may also +// accounts. If slots from only one account is requested, an origin marker may also // be used to retrieve from there. func (p *Peer) RequestStorageRanges(id uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, bytes uint64) error { if len(accounts) == 1 && origin != nil { @@ -119,7 +119,7 @@ func (p *Peer) RequestByteCodes(id uint64, hashes []common.Hash, bytes uint64) e } // RequestTrieNodes fetches a batch of account or storage trie nodes rooted in -// a specificstate trie. +// a specific state trie. func (p *Peer) RequestTrieNodes(id uint64, root common.Hash, paths []TrieNodePathSet, bytes uint64) error { p.logger.Trace("Fetching set of trie nodes", "reqid", id, "root", root, "pathsets", len(paths), "bytes", common.StorageSize(bytes)) diff --git a/eth/protocols/snap/sort_test.go b/eth/protocols/snap/sort_test.go index c625be09ea549..be0a8c5706968 100644 --- a/eth/protocols/snap/sort_test.go +++ b/eth/protocols/snap/sort_test.go @@ -22,7 +22,6 @@ import ( "testing" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/trie" ) func hexToNibbles(s string) []byte { @@ -38,22 +37,17 @@ func hexToNibbles(s string) []byte { } func TestRequestSorting(t *testing.T) { - // - Path 0x9 -> {0x19} // - Path 0x99 -> {0x0099} // - Path 0x01234567890123456789012345678901012345678901234567890123456789019 -> {0x0123456789012345678901234567890101234567890123456789012345678901, 0x19} // - Path 0x012345678901234567890123456789010123456789012345678901234567890199 -> {0x0123456789012345678901234567890101234567890123456789012345678901, 0x0099} - var f = func(path string) (trie.SyncPath, TrieNodePathSet, common.Hash) { + var f = func(path string) string { data := hexToNibbles(path) - sp := trie.NewSyncPath(data) - tnps := TrieNodePathSet([][]byte(sp)) - hash := common.Hash{} - return sp, tnps, hash + return string(data) } var ( - hashes []common.Hash - paths []trie.SyncPath - pathsets []TrieNodePathSet + hashes []common.Hash + paths []string ) for _, x := range []string{ "0x9", @@ -67,16 +61,14 @@ func TestRequestSorting(t *testing.T) { "0x01234567890123456789012345678901012345678901234567890123456789010", "0x01234567890123456789012345678901012345678901234567890123456789011", } { - sp, tnps, hash := f(x) - hashes = append(hashes, hash) - paths = append(paths, sp) - pathsets = append(pathsets, tnps) + paths = append(paths, f(x)) + hashes = append(hashes, common.Hash{}) } - _, paths, pathsets = sortByAccountPath(hashes, paths) + _, _, syncPaths, pathsets := sortByAccountPath(paths, hashes) { var b = new(bytes.Buffer) - for i := 0; i < len(paths); i++ { - fmt.Fprintf(b, "\n%d. paths %x", i, paths[i]) + for i := 0; i < len(syncPaths); i++ { + fmt.Fprintf(b, "\n%d. paths %x", i, syncPaths[i]) } want := ` 0. paths [0099] diff --git a/eth/protocols/snap/sync.go b/eth/protocols/snap/sync.go index 415253c839be2..f262824f9adff 100644 --- a/eth/protocols/snap/sync.go +++ b/eth/protocols/snap/sync.go @@ -21,10 +21,12 @@ import ( "encoding/json" "errors" "fmt" + gomath "math" "math/big" "math/rand" "sort" "sync" + "sync/atomic" "time" "github.com/ethereum/go-ethereum/common" @@ -78,6 +80,29 @@ const ( // and waste round trip times. If it's too high, we're capping responses and // waste bandwidth. maxTrieRequestCount = maxRequestSize / 512 + + // trienodeHealRateMeasurementImpact is the impact a single measurement has on + // the local node's trienode processing capacity. A value closer to 0 reacts + // slower to sudden changes, but it is also more stable against temporary hiccups. + trienodeHealRateMeasurementImpact = 0.005 + + // minTrienodeHealThrottle is the minimum divisor for throttling trie node + // heal requests to avoid overloading the local node and exessively expanding + // the state trie bedth wise. + minTrienodeHealThrottle = 1 + + // maxTrienodeHealThrottle is the maximum divisor for throttling trie node + // heal requests to avoid overloading the local node and exessively expanding + // the state trie bedth wise. + maxTrienodeHealThrottle = maxTrieRequestCount + + // trienodeHealThrottleIncrease is the multiplier for the throttle when the + // rate of arriving data is higher than the rate of processing it. + trienodeHealThrottleIncrease = 1.33 + + // trienodeHealThrottleDecrease is the divisor for the throttle when the + // rate of arriving data is lower than the rate of processing it. + trienodeHealThrottleDecrease = 1.25 ) var ( @@ -230,8 +255,8 @@ type trienodeHealRequest struct { timeout *time.Timer // Timer to track delivery timeout stale chan struct{} // Channel to signal the request was dropped - hashes []common.Hash // Trie node hashes to validate responses - paths []trie.SyncPath // Trie node paths requested for rescheduling + paths []string // Trie node paths for identifying trie node + hashes []common.Hash // Trie node hashes to validate responses task *healTask // Task which this request is filling (only access fields through the runloop!!) } @@ -240,9 +265,9 @@ type trienodeHealRequest struct { type trienodeHealResponse struct { task *healTask // Task which this request is filling - hashes []common.Hash // Hashes of the trie nodes to avoid double hashing - paths []trie.SyncPath // Trie node paths requested for rescheduling missing ones - nodes [][]byte // Actual trie nodes to store into the database (nil = missing) + paths []string // Paths of the trie nodes + hashes []common.Hash // Hashes of the trie nodes to avoid double hashing + nodes [][]byte // Actual trie nodes to store into the database (nil = missing) } // bytecodeHealRequest tracks a pending bytecode request to ensure responses are to @@ -321,8 +346,8 @@ type storageTask struct { type healTask struct { scheduler *trie.Sync // State trie sync scheduler defining the tasks - trieTasks map[common.Hash]trie.SyncPath // Set of trie node tasks currently queued for retrieval - codeTasks map[common.Hash]struct{} // Set of byte code tasks currently queued for retrieval + trieTasks map[string]common.Hash // Set of trie node tasks currently queued for retrieval, indexed by node path + codeTasks map[common.Hash]struct{} // Set of byte code tasks currently queued for retrieval, indexed by code hash } // SyncProgress is a database entry to allow suspending and resuming a snapshot state @@ -365,7 +390,7 @@ type SyncPeer interface { RequestAccountRange(id uint64, root, origin, limit common.Hash, bytes uint64) error // RequestStorageRanges fetches a batch of storage slots belonging to one or - // more accounts. If slots from only one accout is requested, an origin marker + // more accounts. If slots from only one account is requested, an origin marker // may also be used to retrieve from there. RequestStorageRanges(id uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, bytes uint64) error @@ -373,7 +398,7 @@ type SyncPeer interface { RequestByteCodes(id uint64, hashes []common.Hash, bytes uint64) error // RequestTrieNodes fetches a batch of account or storage trie nodes rooted in - // a specificstate trie. + // a specific state trie. RequestTrieNodes(id uint64, root common.Hash, paths []TrieNodePathSet, bytes uint64) error // Log retrieves the peer's own contextual logger. @@ -431,6 +456,11 @@ type Syncer struct { trienodeHealReqs map[uint64]*trienodeHealRequest // Trie node requests currently running bytecodeHealReqs map[uint64]*bytecodeHealRequest // Bytecode requests currently running + trienodeHealRate float64 // Average heal rate for processing trie node data + trienodeHealPend uint64 // Number of trie nodes currently pending for processing + trienodeHealThrottle float64 // Divisor for throttling the amount of trienode heal data requested + trienodeHealThrottled time.Time // Timestamp the last time the throttle was updated + trienodeHealSynced uint64 // Number of state trie nodes downloaded trienodeHealBytes common.StorageSize // Number of state trie bytes persisted to disk trienodeHealDups uint64 // Number of state trie nodes already processed @@ -476,9 +506,10 @@ func NewSyncer(db ethdb.KeyValueStore) *Syncer { trienodeHealIdlers: make(map[string]struct{}), bytecodeHealIdlers: make(map[string]struct{}), - trienodeHealReqs: make(map[uint64]*trienodeHealRequest), - bytecodeHealReqs: make(map[uint64]*bytecodeHealRequest), - stateWriter: db.NewBatch(), + trienodeHealReqs: make(map[uint64]*trienodeHealRequest), + bytecodeHealReqs: make(map[uint64]*bytecodeHealRequest), + trienodeHealThrottle: maxTrienodeHealThrottle, // Tune downward instead of insta-filling with junk + stateWriter: db.NewBatch(), extProgress: new(SyncProgress), } @@ -540,7 +571,7 @@ func (s *Syncer) Unregister(id string) error { return nil } -// Sync starts (or resumes a previous) sync cycle to iterate over an state trie +// Sync starts (or resumes a previous) sync cycle to iterate over a state trie // with the given root and reconstruct the nodes based on the snapshot leaves. // Previously downloaded segments will not be redownloaded of fixed, rather any // errors will be healed after the leaves are fully accumulated. @@ -551,7 +582,7 @@ func (s *Syncer) Sync(root common.Hash, cancel chan struct{}) error { s.root = root s.healer = &healTask{ scheduler: state.NewStateSync(root, s.db, s.onHealState), - trieTasks: make(map[common.Hash]trie.SyncPath), + trieTasks: make(map[string]common.Hash), codeTasks: make(map[common.Hash]struct{}), } s.statelessPeers = make(map[string]struct{}) @@ -584,6 +615,8 @@ func (s *Syncer) Sync(root common.Hash, cancel chan struct{}) error { } }() defer s.report(true) + // commit any trie- and bytecode-healing data. + defer s.commitHealer(true) // Whether sync completed or not, disregard any future packets defer func() { @@ -712,7 +745,7 @@ func (s *Syncer) loadSyncStatus() { } task.genTrie = trie.NewStackTrie(task.genBatch) - for _, subtasks := range task.SubTasks { + for accountHash, subtasks := range task.SubTasks { for _, subtask := range subtasks { subtask.genBatch = ethdb.HookedBatch{ Batch: s.db.NewBatch(), @@ -720,7 +753,7 @@ func (s *Syncer) loadSyncStatus() { s.storageBytes += common.StorageSize(len(key) + len(value)) }, } - subtask.genTrie = trie.NewStackTrie(subtask.genBatch) + subtask.genTrie = trie.NewStackTrieWithOwner(subtask.genBatch, accountHash) } } } @@ -743,7 +776,7 @@ func (s *Syncer) loadSyncStatus() { return } } - // Either we've failed to decode the previus state, or there was none. + // Either we've failed to decode the previous state, or there was none. // Start a fresh sync by chunking up the account range and scheduling // them for retrieval. s.tasks = nil @@ -1183,10 +1216,10 @@ func (s *Syncer) assignStorageTasks(success chan *storageResponse, fail chan *st } if subtask == nil { // No large contract required retrieval, but small ones available - for acccount, root := range task.stateTasks { - delete(task.stateTasks, acccount) + for account, root := range task.stateTasks { + delete(task.stateTasks, account) - accounts = append(accounts, acccount) + accounts = append(accounts, account) roots = append(roots, root) if len(accounts) >= storageSets { @@ -1280,9 +1313,9 @@ func (s *Syncer) assignTrienodeHealTasks(success chan *trienodeHealResponse, fai want = maxTrieRequestCount + maxCodeRequestCount ) if have < want { - nodes, paths, codes := s.healer.scheduler.Missing(want - have) - for i, hash := range nodes { - s.healer.trieTasks[hash] = paths[i] + paths, hashes, codes := s.healer.scheduler.Missing(want - have) + for i, path := range paths { + s.healer.trieTasks[path] = hashes[i] } for _, hash := range codes { s.healer.codeTasks[hash] = struct{}{} @@ -1321,23 +1354,26 @@ func (s *Syncer) assignTrienodeHealTasks(success chan *trienodeHealResponse, fai if cap > maxTrieRequestCount { cap = maxTrieRequestCount } + cap = int(float64(cap) / s.trienodeHealThrottle) + if cap <= 0 { + cap = 1 + } var ( hashes = make([]common.Hash, 0, cap) - paths = make([]trie.SyncPath, 0, cap) + paths = make([]string, 0, cap) pathsets = make([]TrieNodePathSet, 0, cap) ) - for hash, pathset := range s.healer.trieTasks { - delete(s.healer.trieTasks, hash) + for path, hash := range s.healer.trieTasks { + delete(s.healer.trieTasks, path) + paths = append(paths, path) hashes = append(hashes, hash) - paths = append(paths, pathset) - - if len(hashes) >= cap { + if len(paths) >= cap { break } } // Group requests by account hash - hashes, paths, pathsets = sortByAccountPath(hashes, paths) + paths, hashes, _, pathsets = sortByAccountPath(paths, hashes) req := &trienodeHealRequest{ peer: idle, id: reqid, @@ -1346,8 +1382,8 @@ func (s *Syncer) assignTrienodeHealTasks(success chan *trienodeHealResponse, fai revert: fail, cancel: cancel, stale: make(chan struct{}), - hashes: hashes, paths: paths, + hashes: hashes, task: s.healer, } req.timeout = time.AfterFunc(s.rates.TargetTimeout(), func() { @@ -1405,9 +1441,9 @@ func (s *Syncer) assignBytecodeHealTasks(success chan *bytecodeHealResponse, fai want = maxTrieRequestCount + maxCodeRequestCount ) if have < want { - nodes, paths, codes := s.healer.scheduler.Missing(want - have) - for i, hash := range nodes { - s.healer.trieTasks[hash] = paths[i] + paths, hashes, codes := s.healer.scheduler.Missing(want - have) + for i, path := range paths { + s.healer.trieTasks[path] = hashes[i] } for _, hash := range codes { s.healer.codeTasks[hash] = struct{}{} @@ -1487,7 +1523,7 @@ func (s *Syncer) assignBytecodeHealTasks(success chan *bytecodeHealResponse, fai } } -// revertRequests locates all the currently pending reuqests from a particular +// revertRequests locates all the currently pending requests from a particular // peer and reverts them, rescheduling for others to fulfill. func (s *Syncer) revertRequests(peer string) { // Gather the requests first, revertals need the lock too @@ -1576,7 +1612,7 @@ func (s *Syncer) revertAccountRequest(req *accountRequest) { s.lock.Unlock() // If there's a timeout timer still running, abort it and mark the account - // task as not-pending, ready for resheduling + // task as not-pending, ready for rescheduling req.timeout.Stop() if req.task.req == req { req.task.req = nil @@ -1617,7 +1653,7 @@ func (s *Syncer) revertBytecodeRequest(req *bytecodeRequest) { s.lock.Unlock() // If there's a timeout timer still running, abort it and mark the code - // retrievals as not-pending, ready for resheduling + // retrievals as not-pending, ready for rescheduling req.timeout.Stop() for _, hash := range req.hashes { req.task.codeTasks[hash] = struct{}{} @@ -1658,7 +1694,7 @@ func (s *Syncer) revertStorageRequest(req *storageRequest) { s.lock.Unlock() // If there's a timeout timer still running, abort it and mark the storage - // task as not-pending, ready for resheduling + // task as not-pending, ready for rescheduling req.timeout.Stop() if req.subTask != nil { req.subTask.req = nil @@ -1703,10 +1739,10 @@ func (s *Syncer) revertTrienodeHealRequest(req *trienodeHealRequest) { s.lock.Unlock() // If there's a timeout timer still running, abort it and mark the trie node - // retrievals as not-pending, ready for resheduling + // retrievals as not-pending, ready for rescheduling req.timeout.Stop() - for i, hash := range req.hashes { - req.task.trieTasks[hash] = req.paths[i] + for i, path := range req.paths { + req.task.trieTasks[path] = req.hashes[i] } } @@ -1744,7 +1780,7 @@ func (s *Syncer) revertBytecodeHealRequest(req *bytecodeHealRequest) { s.lock.Unlock() // If there's a timeout timer still running, abort it and mark the code - // retrievals as not-pending, ready for resheduling + // retrievals as not-pending, ready for rescheduling req.timeout.Stop() for _, hash := range req.hashes { req.task.codeTasks[hash] = struct{}{} @@ -1971,7 +2007,7 @@ func (s *Syncer) processStorageResponse(res *storageResponse) { Last: r.End(), root: acc.Root, genBatch: batch, - genTrie: trie.NewStackTrie(batch), + genTrie: trie.NewStackTrieWithOwner(batch, account), }) for r.Next() { batch := ethdb.HookedBatch{ @@ -1985,7 +2021,7 @@ func (s *Syncer) processStorageResponse(res *storageResponse) { Last: r.End(), root: acc.Root, genBatch: batch, - genTrie: trie.NewStackTrie(batch), + genTrie: trie.NewStackTrieWithOwner(batch, account), }) } for _, task := range tasks { @@ -2030,13 +2066,13 @@ func (s *Syncer) processStorageResponse(res *storageResponse) { slots += len(res.hashes[i]) if i < len(res.hashes)-1 || res.subTask == nil { - tr := trie.NewStackTrie(batch) + tr := trie.NewStackTrieWithOwner(batch, account) for j := 0; j < len(res.hashes[i]); j++ { tr.Update(res.hashes[i][j][:], res.slots[i][j]) } tr.Commit() } - // Persist the received storage segements. These flat state maybe + // Persist the received storage segments. These flat state maybe // outdated during the sync, but it can be fixed later during the // snapshot generation. for j := 0; j < len(res.hashes[i]); j++ { @@ -2091,19 +2127,25 @@ func (s *Syncer) processStorageResponse(res *storageResponse) { // processTrienodeHealResponse integrates an already validated trienode response // into the healer tasks. func (s *Syncer) processTrienodeHealResponse(res *trienodeHealResponse) { + var ( + start = time.Now() + fills int + ) for i, hash := range res.hashes { node := res.nodes[i] // If the trie node was not delivered, reschedule it if node == nil { - res.task.trieTasks[hash] = res.paths[i] + res.task.trieTasks[res.paths[i]] = res.hashes[i] continue } + fills++ + // Push the trie node into the state syncer s.trienodeHealSynced++ s.trienodeHealBytes += common.StorageSize(len(node)) - err := s.healer.scheduler.Process(trie.SyncResult{Hash: hash, Data: node}) + err := s.healer.scheduler.ProcessNode(trie.NodeSyncResult{Path: res.paths[i], Data: node}) switch err { case nil: case trie.ErrAlreadyProcessed: @@ -2114,6 +2156,57 @@ func (s *Syncer) processTrienodeHealResponse(res *trienodeHealResponse) { log.Error("Invalid trienode processed", "hash", hash, "err", err) } } + s.commitHealer(false) + + // Calculate the processing rate of one filled trie node + rate := float64(fills) / (float64(time.Since(start)) / float64(time.Second)) + + // Update the currently measured trienode queueing and processing throughput. + // + // The processing rate needs to be updated uniformly independent if we've + // processed 1x100 trie nodes or 100x1 to keep the rate consistent even in + // the face of varying network packets. As such, we cannot just measure the + // time it took to process N trie nodes and update once, we need one update + // per trie node. + // + // Naively, that would be: + // + // for i:=0; i time.Second { + // Periodically adjust the trie node throttler + if float64(pending) > 2*s.trienodeHealRate { + s.trienodeHealThrottle *= trienodeHealThrottleIncrease + } else { + s.trienodeHealThrottle /= trienodeHealThrottleDecrease + } + if s.trienodeHealThrottle > maxTrienodeHealThrottle { + s.trienodeHealThrottle = maxTrienodeHealThrottle + } else if s.trienodeHealThrottle < minTrienodeHealThrottle { + s.trienodeHealThrottle = minTrienodeHealThrottle + } + s.trienodeHealThrottled = time.Now() + + log.Debug("Updated trie node heal throttler", "rate", s.trienodeHealRate, "pending", pending, "throttle", s.trienodeHealThrottle) + } +} + +func (s *Syncer) commitHealer(force bool) { + if !force && s.healer.scheduler.MemSize() < ethdb.IdealBatchSize { + return + } batch := s.db.NewBatch() if err := s.healer.scheduler.Commit(batch); err != nil { log.Error("Failed to commit healing data", "err", err) @@ -2139,7 +2232,7 @@ func (s *Syncer) processBytecodeHealResponse(res *bytecodeHealResponse) { s.bytecodeHealSynced++ s.bytecodeHealBytes += common.StorageSize(len(node)) - err := s.healer.scheduler.Process(trie.SyncResult{Hash: hash, Data: node}) + err := s.healer.scheduler.ProcessCode(trie.CodeSyncResult{Hash: hash, Data: node}) switch err { case nil: case trie.ErrAlreadyProcessed: @@ -2150,14 +2243,7 @@ func (s *Syncer) processBytecodeHealResponse(res *bytecodeHealResponse) { log.Error("Invalid bytecode processed", "hash", hash, "err", err) } } - batch := s.db.NewBatch() - if err := s.healer.scheduler.Commit(batch); err != nil { - log.Error("Failed to commit healing data", "err", err) - } - if err := batch.Write(); err != nil { - log.Crit("Failed to persist healing data", "err", err) - } - log.Debug("Persisted set of healing data", "type", "bytecode", "bytes", common.StorageSize(batch.ValueSize())) + s.commitHealer(false) } // forwardAccountTask takes a filled account task and persists anything available @@ -2171,7 +2257,7 @@ func (s *Syncer) forwardAccountTask(task *accountTask) { } task.res = nil - // Persist the received account segements. These flat state maybe + // Persist the received account segments. These flat state maybe // outdated during the sync, but it can be fixed later during the // snapshot generation. oldAccountBytes := s.accountBytes @@ -2249,14 +2335,18 @@ func (s *Syncer) OnAccounts(peer SyncPeer, id uint64, hashes []common.Hash, acco // Whether or not the response is valid, we can mark the peer as idle and // notify the scheduler to assign a new task. If the response is invalid, // we'll drop the peer in a bit. + defer func() { + s.lock.Lock() + defer s.lock.Unlock() + if _, ok := s.peers[peer.ID()]; ok { + s.accountIdlers[peer.ID()] = struct{}{} + } + select { + case s.update <- struct{}{}: + default: + } + }() s.lock.Lock() - if _, ok := s.peers[peer.ID()]; ok { - s.accountIdlers[peer.ID()] = struct{}{} - } - select { - case s.update <- struct{}{}: - default: - } // Ensure the response is for a valid request req, ok := s.accountReqs[id] if !ok { @@ -2361,14 +2451,18 @@ func (s *Syncer) onByteCodes(peer SyncPeer, id uint64, bytecodes [][]byte) error // Whether or not the response is valid, we can mark the peer as idle and // notify the scheduler to assign a new task. If the response is invalid, // we'll drop the peer in a bit. + defer func() { + s.lock.Lock() + defer s.lock.Unlock() + if _, ok := s.peers[peer.ID()]; ok { + s.bytecodeIdlers[peer.ID()] = struct{}{} + } + select { + case s.update <- struct{}{}: + default: + } + }() s.lock.Lock() - if _, ok := s.peers[peer.ID()]; ok { - s.bytecodeIdlers[peer.ID()] = struct{}{} - } - select { - case s.update <- struct{}{}: - default: - } // Ensure the response is for a valid request req, ok := s.bytecodeReqs[id] if !ok { @@ -2470,14 +2564,18 @@ func (s *Syncer) OnStorage(peer SyncPeer, id uint64, hashes [][]common.Hash, slo // Whether or not the response is valid, we can mark the peer as idle and // notify the scheduler to assign a new task. If the response is invalid, // we'll drop the peer in a bit. + defer func() { + s.lock.Lock() + defer s.lock.Unlock() + if _, ok := s.peers[peer.ID()]; ok { + s.storageIdlers[peer.ID()] = struct{}{} + } + select { + case s.update <- struct{}{}: + default: + } + }() s.lock.Lock() - if _, ok := s.peers[peer.ID()]; ok { - s.storageIdlers[peer.ID()] = struct{}{} - } - select { - case s.update <- struct{}{}: - default: - } // Ensure the response is for a valid request req, ok := s.storageReqs[id] if !ok { @@ -2597,14 +2695,18 @@ func (s *Syncer) OnTrieNodes(peer SyncPeer, id uint64, trienodes [][]byte) error // Whether or not the response is valid, we can mark the peer as idle and // notify the scheduler to assign a new task. If the response is invalid, // we'll drop the peer in a bit. + defer func() { + s.lock.Lock() + defer s.lock.Unlock() + if _, ok := s.peers[peer.ID()]; ok { + s.trienodeHealIdlers[peer.ID()] = struct{}{} + } + select { + case s.update <- struct{}{}: + default: + } + }() s.lock.Lock() - if _, ok := s.peers[peer.ID()]; ok { - s.trienodeHealIdlers[peer.ID()] = struct{}{} - } - select { - case s.update <- struct{}{}: - default: - } // Ensure the response is for a valid request req, ok := s.trienodeHealReqs[id] if !ok { @@ -2640,10 +2742,12 @@ func (s *Syncer) OnTrieNodes(peer SyncPeer, id uint64, trienodes [][]byte) error // Cross reference the requested trienodes with the response to find gaps // that the serving node is missing - hasher := sha3.NewLegacyKeccak256().(crypto.KeccakState) - hash := make([]byte, 32) - - nodes := make([][]byte, len(req.hashes)) + var ( + hasher = sha3.NewLegacyKeccak256().(crypto.KeccakState) + hash = make([]byte, 32) + nodes = make([][]byte, len(req.hashes)) + fills uint64 + ) for i, j := 0, 0; i < len(trienodes); i++ { // Find the next hash that we've been served, leaving misses with nils hasher.Reset() @@ -2655,20 +2759,26 @@ func (s *Syncer) OnTrieNodes(peer SyncPeer, id uint64, trienodes [][]byte) error } if j < len(req.hashes) { nodes[j] = trienodes[i] + fills++ j++ continue } // We've either ran out of hashes, or got unrequested data logger.Warn("Unexpected healing trienodes", "count", len(trienodes)-i) + // Signal this request as failed, and ready for rescheduling s.scheduleRevertTrienodeHealRequest(req) return errors.New("unexpected healing trienode") } // Response validated, send it to the scheduler for filling + atomic.AddUint64(&s.trienodeHealPend, fills) + defer func() { + atomic.AddUint64(&s.trienodeHealPend, ^(fills - 1)) + }() response := &trienodeHealResponse{ + paths: req.paths, task: req.task, hashes: req.hashes, - paths: req.paths, nodes: nodes, } select { @@ -2692,14 +2802,18 @@ func (s *Syncer) onHealByteCodes(peer SyncPeer, id uint64, bytecodes [][]byte) e // Whether or not the response is valid, we can mark the peer as idle and // notify the scheduler to assign a new task. If the response is invalid, // we'll drop the peer in a bit. + defer func() { + s.lock.Lock() + defer s.lock.Unlock() + if _, ok := s.peers[peer.ID()]; ok { + s.bytecodeHealIdlers[peer.ID()] = struct{}{} + } + select { + case s.update <- struct{}{}: + default: + } + }() s.lock.Lock() - if _, ok := s.peers[peer.ID()]; ok { - s.bytecodeHealIdlers[peer.ID()] = struct{}{} - } - select { - case s.update <- struct{}{}: - default: - } // Ensure the response is for a valid request req, ok := s.bytecodeHealReqs[id] if !ok { @@ -2774,14 +2888,14 @@ func (s *Syncer) onHealByteCodes(peer SyncPeer, id uint64, bytecodes [][]byte) e } // onHealState is a callback method to invoke when a flat state(account -// or storage slot) is downloded during the healing stage. The flat states +// or storage slot) is downloaded during the healing stage. The flat states // can be persisted blindly and can be fixed later in the generation stage. // Note it's not concurrent safe, please handle the concurrent issue outside. func (s *Syncer) onHealState(paths [][]byte, value []byte) error { if len(paths) == 1 { var account types.StateAccount if err := rlp.DecodeBytes(value, &account); err != nil { - return nil + return nil // Returning the error here would drop the remote peer } blob := snapshot.SlimAccountRLP(account.Nonce, account.Balance, account.Root, account.CodeHash) rawdb.WriteAccountSnapshot(s.stateWriter, common.BytesToHash(paths[0]), blob) @@ -2850,7 +2964,7 @@ func (s *Syncer) reportSyncProgress(force bool) { storage = fmt.Sprintf("%v@%v", log.FormatLogfmtUint64(s.storageSynced), s.storageBytes.TerminalString()) bytecode = fmt.Sprintf("%v@%v", log.FormatLogfmtUint64(s.bytecodeSynced), s.bytecodeBytes.TerminalString()) ) - log.Info("State sync in progress", "synced", progress, "state", synced, + log.Info("Syncing: state download in progress", "synced", progress, "state", synced, "accounts", accounts, "slots", storage, "codes", bytecode, "eta", common.PrettyDuration(estTime-elapsed)) } @@ -2869,7 +2983,7 @@ func (s *Syncer) reportHealProgress(force bool) { accounts = fmt.Sprintf("%v@%v", log.FormatLogfmtUint64(s.accountHealed), s.accountHealedBytes.TerminalString()) storage = fmt.Sprintf("%v@%v", log.FormatLogfmtUint64(s.storageHealed), s.storageHealedBytes.TerminalString()) ) - log.Info("State heal in progress", "accounts", accounts, "slots", storage, + log.Info("Syncing: state healing in progress", "accounts", accounts, "slots", storage, "codes", bytecode, "nodes", trienode, "pending", s.healer.scheduler.Pending()) } @@ -2913,8 +3027,9 @@ func (s *capacitySort) Swap(i, j int) { // healRequestSort implements the Sort interface, allowing sorting trienode // heal requests, which is a prerequisite for merging storage-requests. type healRequestSort struct { - hashes []common.Hash - paths []trie.SyncPath + paths []string + hashes []common.Hash + syncPaths []trie.SyncPath } func (t *healRequestSort) Len() int { @@ -2922,8 +3037,8 @@ func (t *healRequestSort) Len() int { } func (t *healRequestSort) Less(i, j int) bool { - a := t.paths[i] - b := t.paths[j] + a := t.syncPaths[i] + b := t.syncPaths[j] switch bytes.Compare(a[0], b[0]) { case -1: return true @@ -2944,8 +3059,9 @@ func (t *healRequestSort) Less(i, j int) bool { } func (t *healRequestSort) Swap(i, j int) { - t.hashes[i], t.hashes[j] = t.hashes[j], t.hashes[i] t.paths[i], t.paths[j] = t.paths[j], t.paths[i] + t.hashes[i], t.hashes[j] = t.hashes[j], t.hashes[i] + t.syncPaths[i], t.syncPaths[j] = t.syncPaths[j], t.syncPaths[i] } // Merge merges the pathsets, so that several storage requests concerning the @@ -2953,7 +3069,7 @@ func (t *healRequestSort) Swap(i, j int) { // OBS: This operation is moot if t has not first been sorted. func (t *healRequestSort) Merge() []TrieNodePathSet { var result []TrieNodePathSet - for _, path := range t.paths { + for _, path := range t.syncPaths { pathset := TrieNodePathSet([][]byte(path)) if len(path) == 1 { // It's an account reference. @@ -2962,7 +3078,7 @@ func (t *healRequestSort) Merge() []TrieNodePathSet { // It's a storage reference. end := len(result) - 1 if len(result) == 0 || !bytes.Equal(pathset[0], result[end][0]) { - // The account doesn't doesn't match last, create a new entry. + // The account doesn't match last, create a new entry. result = append(result, pathset) } else { // It's the same account as the previous one, add to the storage @@ -2976,9 +3092,13 @@ func (t *healRequestSort) Merge() []TrieNodePathSet { // sortByAccountPath takes hashes and paths, and sorts them. After that, it generates // the TrieNodePaths and merges paths which belongs to the same account path. -func sortByAccountPath(hashes []common.Hash, paths []trie.SyncPath) ([]common.Hash, []trie.SyncPath, []TrieNodePathSet) { - n := &healRequestSort{hashes, paths} +func sortByAccountPath(paths []string, hashes []common.Hash) ([]string, []common.Hash, []trie.SyncPath, []TrieNodePathSet) { + var syncPaths []trie.SyncPath + for _, path := range paths { + syncPaths = append(syncPaths, trie.NewSyncPath([]byte(path))) + } + n := &healRequestSort{paths, hashes, syncPaths} sort.Sort(n) pathsets := n.Merge() - return n.hashes, n.paths, pathsets + return n.paths, n.hashes, n.syncPaths, pathsets } diff --git a/eth/protocols/snap/sync_test.go b/eth/protocols/snap/sync_test.go index e727544fa4500..1d1ce932e073d 100644 --- a/eth/protocols/snap/sync_test.go +++ b/eth/protocols/snap/sync_test.go @@ -368,8 +368,8 @@ func createStorageRequestResponse(t *testPeer, root common.Hash, accounts []comm return hashes, slots, proofs } -// the createStorageRequestResponseAlwaysProve tests a cornercase, where it always -// supplies the proof for the last account, even if it is 'complete'.h +// createStorageRequestResponseAlwaysProve tests a cornercase, where the peer always +// supplies the proof for the last account, even if it is 'complete'. func createStorageRequestResponseAlwaysProve(t *testPeer, root common.Hash, accounts []common.Hash, bOrigin, bLimit []byte, max uint64) (hashes [][]common.Hash, slots [][][]byte, proofs [][]byte) { var size uint64 max = max * 3 / 4 @@ -1348,9 +1348,11 @@ func getCodeByHash(hash common.Hash) []byte { // makeAccountTrieNoStorage spits out a trie, along with the leafs func makeAccountTrieNoStorage(n int) (*trie.Trie, entrySlice) { - db := trie.NewDatabase(rawdb.NewMemoryDatabase()) - accTrie, _ := trie.New(common.Hash{}, db) - var entries entrySlice + var ( + db = trie.NewDatabase(rawdb.NewMemoryDatabase()) + accTrie = trie.NewEmpty(db) + entries entrySlice + ) for i := uint64(1); i <= uint64(n); i++ { value, _ := rlp.EncodeToBytes(&types.StateAccount{ Nonce: i, @@ -1364,7 +1366,13 @@ func makeAccountTrieNoStorage(n int) (*trie.Trie, entrySlice) { entries = append(entries, elem) } sort.Sort(entries) - accTrie.Commit(nil) + + // Commit the state changes into db and re-create the trie + // for accessing later. + root, nodes, _ := accTrie.Commit(false) + db.Update(trie.NewWithNodeSet(nodes)) + + accTrie, _ = trie.New(trie.StateTrieID(root), db) return accTrie, entries } @@ -1377,7 +1385,7 @@ func makeBoundaryAccountTrie(n int) (*trie.Trie, entrySlice) { boundaries []common.Hash db = trie.NewDatabase(rawdb.NewMemoryDatabase()) - trie, _ = trie.New(common.Hash{}, db) + accTrie = trie.NewEmpty(db) ) // Initialize boundaries var next common.Hash @@ -1404,7 +1412,7 @@ func makeBoundaryAccountTrie(n int) (*trie.Trie, entrySlice) { CodeHash: getCodeHash(uint64(i)), }) elem := &kv{boundaries[i].Bytes(), value} - trie.Update(elem.k, elem.v) + accTrie.Update(elem.k, elem.v) entries = append(entries, elem) } // Fill other accounts if required @@ -1416,12 +1424,18 @@ func makeBoundaryAccountTrie(n int) (*trie.Trie, entrySlice) { CodeHash: getCodeHash(i), }) elem := &kv{key32(i), value} - trie.Update(elem.k, elem.v) + accTrie.Update(elem.k, elem.v) entries = append(entries, elem) } sort.Sort(entries) - trie.Commit(nil) - return trie, entries + + // Commit the state changes into db and re-create the trie + // for accessing later. + root, nodes, _ := accTrie.Commit(false) + db.Update(trie.NewWithNodeSet(nodes)) + + accTrie, _ = trie.New(trie.StateTrieID(root), db) + return accTrie, entries } // makeAccountTrieWithStorageWithUniqueStorage creates an account trie where each accounts @@ -1429,10 +1443,12 @@ func makeBoundaryAccountTrie(n int) (*trie.Trie, entrySlice) { func makeAccountTrieWithStorageWithUniqueStorage(accounts, slots int, code bool) (*trie.Trie, entrySlice, map[common.Hash]*trie.Trie, map[common.Hash]entrySlice) { var ( db = trie.NewDatabase(rawdb.NewMemoryDatabase()) - accTrie, _ = trie.New(common.Hash{}, db) + accTrie = trie.NewEmpty(db) entries entrySlice + storageRoots = make(map[common.Hash]common.Hash) storageTries = make(map[common.Hash]*trie.Trie) storageEntries = make(map[common.Hash]entrySlice) + nodes = trie.NewMergedNodeSet() ) // Create n accounts in the trie for i := uint64(1); i <= uint64(accounts); i++ { @@ -1442,9 +1458,9 @@ func makeAccountTrieWithStorageWithUniqueStorage(accounts, slots int, code bool) codehash = getCodeHash(i) } // Create a storage trie - stTrie, stEntries := makeStorageTrieWithSeed(uint64(slots), i, db) - stRoot := stTrie.Hash() - stTrie.Commit(nil) + stRoot, stNodes, stEntries := makeStorageTrieWithSeed(common.BytesToHash(key), uint64(slots), i, db) + nodes.Merge(stNodes) + value, _ := rlp.EncodeToBytes(&types.StateAccount{ Nonce: i, Balance: big.NewInt(int64(i)), @@ -1455,12 +1471,26 @@ func makeAccountTrieWithStorageWithUniqueStorage(accounts, slots int, code bool) accTrie.Update(elem.k, elem.v) entries = append(entries, elem) - storageTries[common.BytesToHash(key)] = stTrie + storageRoots[common.BytesToHash(key)] = stRoot storageEntries[common.BytesToHash(key)] = stEntries } sort.Sort(entries) - accTrie.Commit(nil) + // Commit account trie + root, set, _ := accTrie.Commit(true) + nodes.Merge(set) + + // Commit gathered dirty nodes into database + db.Update(nodes) + + // Re-create tries with new root + accTrie, _ = trie.New(trie.StateTrieID(root), db) + for i := uint64(1); i <= uint64(accounts); i++ { + key := key32(i) + id := trie.StorageTrieID(root, common.BytesToHash(key), storageRoots[common.BytesToHash(key)]) + trie, _ := trie.New(id, db) + storageTries[common.BytesToHash(key)] = trie + } return accTrie, entries, storageTries, storageEntries } @@ -1468,23 +1498,13 @@ func makeAccountTrieWithStorageWithUniqueStorage(accounts, slots int, code bool) func makeAccountTrieWithStorage(accounts, slots int, code, boundary bool) (*trie.Trie, entrySlice, map[common.Hash]*trie.Trie, map[common.Hash]entrySlice) { var ( db = trie.NewDatabase(rawdb.NewMemoryDatabase()) - accTrie, _ = trie.New(common.Hash{}, db) + accTrie = trie.NewEmpty(db) entries entrySlice + storageRoots = make(map[common.Hash]common.Hash) storageTries = make(map[common.Hash]*trie.Trie) storageEntries = make(map[common.Hash]entrySlice) + nodes = trie.NewMergedNodeSet() ) - // Make a storage trie which we reuse for the whole lot - var ( - stTrie *trie.Trie - stEntries entrySlice - ) - if boundary { - stTrie, stEntries = makeBoundaryStorageTrie(slots, db) - } else { - stTrie, stEntries = makeStorageTrieWithSeed(uint64(slots), 0, db) - } - stRoot := stTrie.Hash() - // Create n accounts in the trie for i := uint64(1); i <= uint64(accounts); i++ { key := key32(i) @@ -1492,6 +1512,19 @@ func makeAccountTrieWithStorage(accounts, slots int, code, boundary bool) (*trie if code { codehash = getCodeHash(i) } + // Make a storage trie + var ( + stRoot common.Hash + stNodes *trie.NodeSet + stEntries entrySlice + ) + if boundary { + stRoot, stNodes, stEntries = makeBoundaryStorageTrie(common.BytesToHash(key), slots, db) + } else { + stRoot, stNodes, stEntries = makeStorageTrieWithSeed(common.BytesToHash(key), uint64(slots), 0, db) + } + nodes.Merge(stNodes) + value, _ := rlp.EncodeToBytes(&types.StateAccount{ Nonce: i, Balance: big.NewInt(int64(i)), @@ -1501,21 +1534,42 @@ func makeAccountTrieWithStorage(accounts, slots int, code, boundary bool) (*trie elem := &kv{key, value} accTrie.Update(elem.k, elem.v) entries = append(entries, elem) + // we reuse the same one for all accounts - storageTries[common.BytesToHash(key)] = stTrie + storageRoots[common.BytesToHash(key)] = stRoot storageEntries[common.BytesToHash(key)] = stEntries } sort.Sort(entries) - stTrie.Commit(nil) - accTrie.Commit(nil) + + // Commit account trie + root, set, _ := accTrie.Commit(true) + nodes.Merge(set) + + // Commit gathered dirty nodes into database + db.Update(nodes) + + // Re-create tries with new root + accTrie, err := trie.New(trie.StateTrieID(root), db) + if err != nil { + panic(err) + } + for i := uint64(1); i <= uint64(accounts); i++ { + key := key32(i) + id := trie.StorageTrieID(root, common.BytesToHash(key), storageRoots[common.BytesToHash(key)]) + trie, err := trie.New(id, db) + if err != nil { + panic(err) + } + storageTries[common.BytesToHash(key)] = trie + } return accTrie, entries, storageTries, storageEntries } // makeStorageTrieWithSeed fills a storage trie with n items, returning the // not-yet-committed trie and the sorted entries. The seeds can be used to ensure // that tries are unique. -func makeStorageTrieWithSeed(n, seed uint64, db *trie.Database) (*trie.Trie, entrySlice) { - trie, _ := trie.New(common.Hash{}, db) +func makeStorageTrieWithSeed(owner common.Hash, n, seed uint64, db *trie.Database) (common.Hash, *trie.NodeSet, entrySlice) { + trie, _ := trie.New(trie.StorageTrieID(common.Hash{}, owner, common.Hash{}), db) var entries entrySlice for i := uint64(1); i <= n; i++ { // store 'x' at slot 'x' @@ -1530,18 +1584,18 @@ func makeStorageTrieWithSeed(n, seed uint64, db *trie.Database) (*trie.Trie, ent entries = append(entries, elem) } sort.Sort(entries) - trie.Commit(nil) - return trie, entries + root, nodes, _ := trie.Commit(false) + return root, nodes, entries } // makeBoundaryStorageTrie constructs a storage trie. Instead of filling // storage slots normally, this function will fill a few slots which have // boundary hash. -func makeBoundaryStorageTrie(n int, db *trie.Database) (*trie.Trie, entrySlice) { +func makeBoundaryStorageTrie(owner common.Hash, n int, db *trie.Database) (common.Hash, *trie.NodeSet, entrySlice) { var ( entries entrySlice boundaries []common.Hash - trie, _ = trie.New(common.Hash{}, db) + trie, _ = trie.New(trie.StorageTrieID(common.Hash{}, owner, common.Hash{}), db) ) // Initialize boundaries var next common.Hash @@ -1581,14 +1635,14 @@ func makeBoundaryStorageTrie(n int, db *trie.Database) (*trie.Trie, entrySlice) entries = append(entries, elem) } sort.Sort(entries) - trie.Commit(nil) - return trie, entries + root, nodes, _ := trie.Commit(false) + return root, nodes, entries } func verifyTrie(db ethdb.KeyValueStore, root common.Hash, t *testing.T) { t.Helper() triedb := trie.NewDatabase(db) - accTrie, err := trie.New(root, triedb) + accTrie, err := trie.New(trie.StateTrieID(root), triedb) if err != nil { t.Fatal(err) } @@ -1606,7 +1660,8 @@ func verifyTrie(db ethdb.KeyValueStore, root common.Hash, t *testing.T) { } accounts++ if acc.Root != emptyRoot { - storeTrie, err := trie.NewSecure(acc.Root, triedb) + id := trie.StorageTrieID(root, common.BytesToHash(accIt.Key), acc.Root) + storeTrie, err := trie.NewStateTrie(id, triedb) if err != nil { t.Fatal(err) } @@ -1661,7 +1716,7 @@ func TestSyncAccountPerformance(t *testing.T) { // Doing so would bring this number down to zero in this artificial testcase, // but only add extra IO for no reason in practice. if have, want := src.nTrienodeRequests, 1; have != want { - fmt.Printf(src.Stats()) + fmt.Print(src.Stats()) t.Errorf("trie node heal requests wrong, want %d, have %d", want, have) } } diff --git a/eth/state_accessor.go b/eth/state_accessor.go index f01db93a67853..ca59024aed48d 100644 --- a/eth/state_accessor.go +++ b/eth/state_accessor.go @@ -26,39 +26,59 @@ import ( "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/eth/tracers" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/trie" ) +// noopReleaser is returned in case there is no operation expected +// for releasing state. +var noopReleaser = tracers.StateReleaseFunc(func() {}) + // StateAtBlock retrieves the state database associated with a certain block. // If no state is locally available for the given block, a number of blocks // are attempted to be reexecuted to generate the desired state. The optional -// base layer statedb can be passed then it's regarded as the statedb of the +// base layer statedb can be provided which is regarded as the statedb of the // parent block. +// +// An additional release function will be returned if the requested state is +// available. Release is expected to be invoked when the returned state is no longer needed. +// Its purpose is to prevent resource leaking. Though it can be noop in some cases. +// // Parameters: -// - block: The block for which we want the state (== state at the stateRoot of the parent) -// - reexec: The maximum number of blocks to reprocess trying to obtain the desired state -// - base: If the caller is tracing multiple blocks, the caller can provide the parent state -// continuously from the callsite. -// - checklive: if true, then the live 'blockchain' state database is used. If the caller want to -// perform Commit or other 'save-to-disk' changes, this should be set to false to avoid -// storing trash persistently -// - preferDisk: this arg can be used by the caller to signal that even though the 'base' is provided, -// it would be preferrable to start from a fresh state, if we have it on disk. -func (eth *Ethereum) StateAtBlock(block *types.Block, reexec uint64, base *state.StateDB, checkLive bool, preferDisk bool) (statedb *state.StateDB, err error) { +// - block: The block for which we want the state(state = block.Root) +// - reexec: The maximum number of blocks to reprocess trying to obtain the desired state +// - base: If the caller is tracing multiple blocks, the caller can provide the parent +// state continuously from the callsite. +// - readOnly: If true, then the live 'blockchain' state database is used. No mutation should +// be made from caller, e.g. perform Commit or other 'save-to-disk' changes. +// Otherwise, the trash generated by caller may be persisted permanently. +// - preferDisk: this arg can be used by the caller to signal that even though the 'base' is +// provided, it would be preferable to start from a fresh state, if we have it +// on disk. +func (eth *Ethereum) StateAtBlock(block *types.Block, reexec uint64, base *state.StateDB, readOnly bool, preferDisk bool) (statedb *state.StateDB, release tracers.StateReleaseFunc, err error) { var ( current *types.Block database state.Database report = true origin = block.NumberU64() ) - // Check the live database first if we have the state fully available, use that. - if checkLive { - statedb, err = eth.blockchain.StateAt(block.Root()) - if err == nil { - return statedb, nil + // The state is only for reading purposes, check the state presence in + // live database. + if readOnly { + // The state is available in live database, create a reference + // on top to prevent garbage collection and return a release + // function to deref it. + if statedb, err = eth.blockchain.StateAt(block.Root()); err == nil { + statedb.Database().TrieDB().Reference(block.Root(), common.Hash{}) + return statedb, func() { + statedb.Database().TrieDB().Dereference(block.Root()) + }, nil } } + // The state is both for reading and writing, or it's unavailable in disk, + // try to construct/recover the state over an ephemeral trie.Database for + // isolating the live one. if base != nil { if preferDisk { // Create an ephemeral trie.Database for isolating the live one. Otherwise @@ -66,37 +86,37 @@ func (eth *Ethereum) StateAtBlock(block *types.Block, reexec uint64, base *state database = state.NewDatabaseWithConfig(eth.chainDb, &trie.Config{Cache: 16}) if statedb, err = state.New(block.Root(), database, nil); err == nil { log.Info("Found disk backend for state trie", "root", block.Root(), "number", block.Number()) - return statedb, nil + return statedb, noopReleaser, nil } } // The optional base statedb is given, mark the start point as parent block statedb, database, report = base, base.Database(), false current = eth.blockchain.GetBlock(block.ParentHash(), block.NumberU64()-1) } else { - // Otherwise try to reexec blocks until we find a state or reach our limit + // Otherwise, try to reexec blocks until we find a state or reach our limit current = block // Create an ephemeral trie.Database for isolating the live one. Otherwise // the internal junks created by tracing will be persisted into the disk. database = state.NewDatabaseWithConfig(eth.chainDb, &trie.Config{Cache: 16}) - // If we didn't check the dirty database, do check the clean one, otherwise - // we would rewind past a persisted block (specific corner case is chain - // tracing from the genesis). - if !checkLive { + // If we didn't check the live database, do check state over ephemeral database, + // otherwise we would rewind past a persisted block (specific corner case is + // chain tracing from the genesis). + if !readOnly { statedb, err = state.New(current.Root(), database, nil) if err == nil { - return statedb, nil + return statedb, noopReleaser, nil } } // Database does not have the state for the given block, try to regenerate for i := uint64(0); i < reexec; i++ { if current.NumberU64() == 0 { - return nil, errors.New("genesis state is missing") + return nil, nil, errors.New("genesis state is missing") } parent := eth.blockchain.GetBlock(current.ParentHash(), current.NumberU64()-1) if parent == nil { - return nil, fmt.Errorf("missing block %v %d", current.ParentHash(), current.NumberU64()-1) + return nil, nil, fmt.Errorf("missing block %v %d", current.ParentHash(), current.NumberU64()-1) } current = parent @@ -108,13 +128,14 @@ func (eth *Ethereum) StateAtBlock(block *types.Block, reexec uint64, base *state if err != nil { switch err.(type) { case *trie.MissingNodeError: - return nil, fmt.Errorf("required historical state unavailable (reexec=%d)", reexec) + return nil, nil, fmt.Errorf("required historical state unavailable (reexec=%d)", reexec) default: - return nil, err + return nil, nil, err } } } - // State was available at historical point, regenerate + // State is available at historical point, re-execute the blocks on top for + // the desired state. var ( start = time.Now() logged time.Time @@ -129,22 +150,24 @@ func (eth *Ethereum) StateAtBlock(block *types.Block, reexec uint64, base *state // Retrieve the next block to regenerate and process it next := current.NumberU64() + 1 if current = eth.blockchain.GetBlockByNumber(next); current == nil { - return nil, fmt.Errorf("block #%d not found", next) + return nil, nil, fmt.Errorf("block #%d not found", next) } _, _, _, err := eth.blockchain.Processor().Process(current, statedb, vm.Config{}) if err != nil { - return nil, fmt.Errorf("processing block %d failed: %v", current.NumberU64(), err) + return nil, nil, fmt.Errorf("processing block %d failed: %v", current.NumberU64(), err) } // Finalize the state so any modifications are written to the trie root, err := statedb.Commit(eth.blockchain.Config().IsEIP158(current.Number())) if err != nil { - return nil, fmt.Errorf("stateAtBlock commit failed, number %d root %v: %w", + return nil, nil, fmt.Errorf("stateAtBlock commit failed, number %d root %v: %w", current.NumberU64(), current.Root().Hex(), err) } statedb, err = state.New(root, database, nil) if err != nil { - return nil, fmt.Errorf("state reset after block %d failed: %v", current.NumberU64(), err) + return nil, nil, fmt.Errorf("state reset after block %d failed: %v", current.NumberU64(), err) } + // Hold the state reference and also drop the parent state + // to prevent accumulating too many nodes in memory. database.TrieDB().Reference(root, common.Hash{}) if parent != (common.Hash{}) { database.TrieDB().Dereference(parent) @@ -155,28 +178,28 @@ func (eth *Ethereum) StateAtBlock(block *types.Block, reexec uint64, base *state nodes, imgs := database.TrieDB().Size() log.Info("Historical state regenerated", "block", current.NumberU64(), "elapsed", time.Since(start), "nodes", nodes, "preimages", imgs) } - return statedb, nil + return statedb, func() { database.TrieDB().Dereference(block.Root()) }, nil } // stateAtTransaction returns the execution environment of a certain transaction. -func (eth *Ethereum) stateAtTransaction(block *types.Block, txIndex int, reexec uint64) (core.Message, vm.BlockContext, *state.StateDB, error) { +func (eth *Ethereum) stateAtTransaction(block *types.Block, txIndex int, reexec uint64) (core.Message, vm.BlockContext, *state.StateDB, tracers.StateReleaseFunc, error) { // Short circuit if it's genesis block. if block.NumberU64() == 0 { - return nil, vm.BlockContext{}, nil, errors.New("no transaction in genesis") + return nil, vm.BlockContext{}, nil, nil, errors.New("no transaction in genesis") } // Create the parent state database parent := eth.blockchain.GetBlock(block.ParentHash(), block.NumberU64()-1) if parent == nil { - return nil, vm.BlockContext{}, nil, fmt.Errorf("parent %#x not found", block.ParentHash()) + return nil, vm.BlockContext{}, nil, nil, fmt.Errorf("parent %#x not found", block.ParentHash()) } // Lookup the statedb of parent block from the live database, // otherwise regenerate it on the flight. - statedb, err := eth.StateAtBlock(parent, reexec, nil, true, false) + statedb, release, err := eth.StateAtBlock(parent, reexec, nil, true, false) if err != nil { - return nil, vm.BlockContext{}, nil, err + return nil, vm.BlockContext{}, nil, nil, err } if txIndex == 0 && len(block.Transactions()) == 0 { - return nil, vm.BlockContext{}, statedb, nil + return nil, vm.BlockContext{}, statedb, release, nil } // Recompute transactions up to the target index. signer := types.MakeSigner(eth.blockchain.Config(), block.Number()) @@ -186,17 +209,17 @@ func (eth *Ethereum) stateAtTransaction(block *types.Block, txIndex int, reexec txContext := core.NewEVMTxContext(msg) context := core.NewEVMBlockContext(block.Header(), eth.blockchain, nil) if idx == txIndex { - return msg, context, statedb, nil + return msg, context, statedb, release, nil } // Not yet the searched for transaction, execute on top of the current state vmenv := vm.NewEVM(context, txContext, statedb, eth.blockchain.Config(), vm.Config{}) statedb.Prepare(tx.Hash(), idx) if _, err := core.ApplyMessage(vmenv, msg, new(core.GasPool).AddGas(tx.Gas())); err != nil { - return nil, vm.BlockContext{}, nil, fmt.Errorf("transaction %#x failed: %v", tx.Hash(), err) + return nil, vm.BlockContext{}, nil, nil, fmt.Errorf("transaction %#x failed: %v", tx.Hash(), err) } // Ensure any modifications are committed to the state // Only delete empty objects if EIP158/161 (a.k.a Spurious Dragon) is in effect statedb.Finalise(vmenv.ChainConfig().IsEIP158(block.Number())) } - return nil, vm.BlockContext{}, nil, fmt.Errorf("transaction index %d out of range for block %#x", txIndex, block.Hash()) + return nil, vm.BlockContext{}, nil, nil, fmt.Errorf("transaction index %d out of range for block %#x", txIndex, block.Hash()) } diff --git a/eth/sync.go b/eth/sync.go index d67d2311d0d98..8fd86b578cf62 100644 --- a/eth/sync.go +++ b/eth/sync.go @@ -163,7 +163,7 @@ func (cs *chainSyncer) nextSyncOp() *chainSyncOp { // An alternative would be to check the local chain for exceeding the TTD and // avoid triggering a sync in that case, but that could also miss sibling or // other family TTD block being accepted. - if cs.handler.merger.TDDReached() { + if cs.handler.chain.Config().TerminalTotalDifficultyPassed || cs.handler.merger.TDDReached() { return nil } // Ensure we're at minimum peer count. diff --git a/eth/sync_test.go b/eth/sync_test.go index 929a2a9d181c0..0b9f9e1bbaaf7 100644 --- a/eth/sync_test.go +++ b/eth/sync_test.go @@ -30,6 +30,7 @@ import ( // Tests that snap sync is disabled after a successful sync cycle. func TestSnapSyncDisabling66(t *testing.T) { testSnapSyncDisabling(t, eth.ETH66, snap.SNAP1) } +func TestSnapSyncDisabling67(t *testing.T) { testSnapSyncDisabling(t, eth.ETH67, snap.SNAP1) } // Tests that snap sync gets disabled as soon as a real block is successfully // imported into the blockchain. diff --git a/eth/tracers/api.go b/eth/tracers/api.go index 13b89885138e8..1e04bea411f39 100644 --- a/eth/tracers/api.go +++ b/eth/tracers/api.go @@ -20,6 +20,7 @@ import ( "bufio" "bytes" "context" + "encoding/json" "errors" "fmt" "os" @@ -62,6 +63,10 @@ const ( defaultTracechainMemLimit = common.StorageSize(500 * 1024 * 1024) ) +// StateReleaseFunc is used to deallocate resources held by constructing a +// historical state for tracing purposes. +type StateReleaseFunc func() + // Backend interface provides the common API services (that are provided by // both full and light clients) with access to necessary functions. type Backend interface { @@ -74,11 +79,8 @@ type Backend interface { ChainConfig() *params.ChainConfig Engine() consensus.Engine ChainDb() ethdb.Database - // StateAtBlock returns the state corresponding to the stateroot of the block. - // N.B: For executing transactions on block N, the required stateRoot is block N-1, - // so this method should be called with the parent. - StateAtBlock(ctx context.Context, block *types.Block, reexec uint64, base *state.StateDB, checkLive, preferDisk bool) (*state.StateDB, error) - StateAtTransaction(ctx context.Context, block *types.Block, txIndex int, reexec uint64) (core.Message, vm.BlockContext, *state.StateDB, error) + StateAtBlock(ctx context.Context, block *types.Block, reexec uint64, base *state.StateDB, readOnly bool, preferDisk bool) (*state.StateDB, StateReleaseFunc, error) + StateAtTransaction(ctx context.Context, block *types.Block, txIndex int, reexec uint64) (core.Message, vm.BlockContext, *state.StateDB, StateReleaseFunc, error) } // API is the collection of tracing APIs exposed over the private debugging endpoint. @@ -115,7 +117,7 @@ func (context *chainContext) GetHeader(hash common.Hash, number uint64) *types.H return header } -// chainContext construts the context reader which is used by the evm for reading +// chainContext constructs the context reader which is used by the evm for reading // the necessary chain context. func (api *API) chainContext(ctx context.Context) core.ChainContext { return &chainContext{api: api, ctx: ctx} @@ -169,15 +171,15 @@ type TraceConfig struct { Tracer *string Timeout *string Reexec *uint64 + // Config specific to given tracer. Note struct logger + // config are historically embedded in main object. + TracerConfig json.RawMessage } // TraceCallConfig is the config for traceCall API. It holds one more // field to override the state for tracing. type TraceCallConfig struct { - *logger.Config - Tracer *string - Timeout *string - Reexec *uint64 + TraceConfig StateOverrides *ethapi.StateOverride BlockOverrides *ethapi.BlockOverrides } @@ -200,11 +202,11 @@ type txTraceResult struct { type blockTraceTask struct { statedb *state.StateDB // Intermediate state prepped for tracing block *types.Block // Block to trace the transactions from - rootref common.Hash // Trie root reference held for this task - results []*txTraceResult // Trace results procudes by the task + release StateReleaseFunc // The function to release the held resource for this task + results []*txTraceResult // Trace results produced by the task } -// blockTraceResult represets the results of tracing a single block when an entire +// blockTraceResult represents the results of tracing a single block when an entire // chain is being traced. type blockTraceResult struct { Block hexutil.Uint64 `json:"block"` // Block number corresponding to this trace @@ -233,13 +235,6 @@ func (api *API) TraceChain(ctx context.Context, start, end rpc.BlockNumber, conf if from.Number().Cmp(to.Number()) >= 0 { return nil, fmt.Errorf("end block (#%d) needs to come after start block (#%d)", end, start) } - return api.traceChain(ctx, from, to, config) -} - -// traceChain configures a new tracer according to the provided configuration, and -// executes all the transactions contained within. The return value will be one item -// per transaction, dependent on the requested tracer. -func (api *API) traceChain(ctx context.Context, start, end *types.Block, config *TraceConfig) (*rpc.Subscription, error) { // Tracing a chain is a **long** operation, only do with subscriptions notifier, supported := rpc.NotifierFromContext(ctx) if !supported { @@ -247,8 +242,45 @@ func (api *API) traceChain(ctx context.Context, start, end *types.Block, config } sub := notifier.CreateSubscription() - // Prepare all the states for tracing. Note this procedure can take very - // long time. Timeout mechanism is necessary. + resCh := api.traceChain(from, to, config, notifier.Closed()) + go func() { + for result := range resCh { + notifier.Notify(sub.ID, result) + } + }() + return sub, nil +} + +// releaser is a helper tool responsible for caching the release +// callbacks of tracing state. +type releaser struct { + releases []StateReleaseFunc + lock sync.Mutex +} + +func (r *releaser) add(release StateReleaseFunc) { + r.lock.Lock() + defer r.lock.Unlock() + + r.releases = append(r.releases, release) +} + +func (r *releaser) call() { + r.lock.Lock() + defer r.lock.Unlock() + + for _, release := range r.releases { + release() + } + r.releases = r.releases[:0] +} + +// traceChain configures a new tracer according to the provided configuration, and +// executes all the transactions contained within. The tracing chain range includes +// the end block but excludes the start one. The return value will be one item per +// transaction, dependent on the requested tracer. +// The tracing procedure should be aborted in case the closed signal is received. +func (api *API) traceChain(start, end *types.Block, config *TraceConfig, closed <-chan interface{}) chan *blockTraceResult { reexec := defaultTraceReexec if config != nil && config.Reexec != nil { reexec = *config.Reexec @@ -259,20 +291,23 @@ func (api *API) traceChain(ctx context.Context, start, end *types.Block, config threads = blocks } var ( - pend = new(sync.WaitGroup) - tasks = make(chan *blockTraceTask, threads) - results = make(chan *blockTraceTask, threads) - localctx = context.Background() + pend = new(sync.WaitGroup) + ctx = context.Background() + taskCh = make(chan *blockTraceTask, threads) + resCh = make(chan *blockTraceTask, threads) + reler = new(releaser) ) for th := 0; th < threads; th++ { pend.Add(1) go func() { defer pend.Done() - // Fetch and execute the next block trace tasks - for task := range tasks { - signer := types.MakeSigner(api.backend.ChainConfig(), task.block.Number()) - blockCtx := core.NewEVMBlockContext(task.block.Header(), api.chainContext(localctx), nil) + // Fetch and execute the block trace taskCh + for task := range taskCh { + var ( + signer = types.MakeSigner(api.backend.ChainConfig(), task.block.Number()) + blockCtx = core.NewEVMBlockContext(task.block.Header(), api.chainContext(ctx), nil) + ) // Trace all the transactions contained within for i, tx := range task.block.Transactions() { msg, _ := tx.AsMessage(signer, task.block.BaseFee()) @@ -281,7 +316,7 @@ func (api *API) traceChain(ctx context.Context, start, end *types.Block, config TxIndex: i, TxHash: tx.Hash(), } - res, err := api.traceTx(localctx, msg, txctx, blockCtx, task.statedb, config) + res, err := api.traceTx(ctx, msg, txctx, blockCtx, task.statedb, config) if err != nil { task.results[i] = &txTraceResult{Error: err.Error()} log.Warn("Tracing failed", "hash", tx.Hash(), "block", task.block.NumberU64(), "err", err) @@ -291,36 +326,38 @@ func (api *API) traceChain(ctx context.Context, start, end *types.Block, config task.statedb.Finalise(api.backend.ChainConfig().IsEIP158(task.block.Number())) task.results[i] = &txTraceResult{Result: res} } - // Stream the result back to the user or abort on teardown + // Tracing state is used up, queue it for de-referencing + reler.add(task.release) + + // Stream the result back to the result catcher or abort on teardown select { - case results <- task: - case <-notifier.Closed(): + case resCh <- task: + case <-closed: return } } }() } // Start a goroutine to feed all the blocks into the tracers - var ( - begin = time.Now() - derefTodo []common.Hash // list of hashes to dereference from the db - derefsMu sync.Mutex // mutex for the derefs - ) - go func() { var ( logged time.Time + begin = time.Now() number uint64 traced uint64 failed error - parent common.Hash statedb *state.StateDB + release StateReleaseFunc ) // Ensure everything is properly cleaned up on any exit path defer func() { - close(tasks) + close(taskCh) pend.Wait() + // Clean out any pending derefs. + reler.call() + + // Log the chain result switch { case failed != nil: log.Warn("Chain tracing failed", "start", start.NumberU64(), "end", end.NumberU64(), "transactions", traced, "elapsed", time.Since(begin), "err", failed) @@ -329,105 +366,97 @@ func (api *API) traceChain(ctx context.Context, start, end *types.Block, config default: log.Info("Chain tracing finished", "start", start.NumberU64(), "end", end.NumberU64(), "transactions", traced, "elapsed", time.Since(begin)) } - close(results) + close(resCh) }() - var preferDisk bool // Feed all the blocks both into the tracer, as well as fast process concurrently for number = start.NumberU64(); number < end.NumberU64(); number++ { // Stop tracing if interruption was requested select { - case <-notifier.Closed(): + case <-closed: return default: } - // clean out any derefs - derefsMu.Lock() - for _, h := range derefTodo { - statedb.Database().TrieDB().Dereference(h) - } - derefTodo = derefTodo[:0] - derefsMu.Unlock() - // Print progress logs if long enough time elapsed if time.Since(logged) > 8*time.Second { logged = time.Now() log.Info("Tracing chain segment", "start", start.NumberU64(), "end", end.NumberU64(), "current", number, "transactions", traced, "elapsed", time.Since(begin)) } - // Retrieve the parent state to trace on top - block, err := api.blockByNumber(localctx, rpc.BlockNumber(number)) + // Retrieve the parent block and target block for tracing. + block, err := api.blockByNumber(ctx, rpc.BlockNumber(number)) if err != nil { failed = err break } - // Prepare the statedb for tracing. Don't use the live database for - // tracing to avoid persisting state junks into the database. - statedb, err = api.backend.StateAtBlock(localctx, block, reexec, statedb, false, preferDisk) + next, err := api.blockByNumber(ctx, rpc.BlockNumber(number+1)) if err != nil { failed = err break } - if trieDb := statedb.Database().TrieDB(); trieDb != nil { - // Hold the reference for tracer, will be released at the final stage - trieDb.Reference(block.Root(), common.Hash{}) - - // Release the parent state because it's already held by the tracer - if parent != (common.Hash{}) { - trieDb.Dereference(parent) - } - // Prefer disk if the trie db memory grows too much - s1, s2 := trieDb.Size() - if !preferDisk && (s1+s2) > defaultTracechainMemLimit { - log.Info("Switching to prefer-disk mode for tracing", "size", s1+s2) - preferDisk = true - } + // Prepare the statedb for tracing. Don't use the live database for + // tracing to avoid persisting state junks into the database. Switch + // over to `preferDisk` mode only if the memory usage exceeds the + // limit, the trie database will be reconstructed from scratch only + // if the relevant state is available in disk. + var preferDisk bool + if statedb != nil { + s1, s2 := statedb.Database().TrieDB().Size() + preferDisk = s1+s2 > defaultTracechainMemLimit } - parent = block.Root() - - next, err := api.blockByNumber(localctx, rpc.BlockNumber(number+1)) + statedb, release, err = api.backend.StateAtBlock(ctx, block, reexec, statedb, false, preferDisk) if err != nil { failed = err break } + // Clean out any pending derefs. Note this step must be done after + // constructing tracing state, because the tracing state of block + // next depends on the parent state and construction may fail if + // we release too early. + reler.call() + // Send the block over to the concurrent tracers (if not in the fast-forward phase) txs := next.Transactions() select { - case tasks <- &blockTraceTask{statedb: statedb.Copy(), block: next, rootref: block.Root(), results: make([]*txTraceResult, len(txs))}: - case <-notifier.Closed(): + case taskCh <- &blockTraceTask{statedb: statedb.Copy(), block: next, release: release, results: make([]*txTraceResult, len(txs))}: + case <-closed: + reler.add(release) return } traced += uint64(len(txs)) } }() - // Keep reading the trace results and stream the to the user + // Keep reading the trace results and stream them to result channel. + retCh := make(chan *blockTraceResult) go func() { + defer close(retCh) var ( - done = make(map[uint64]*blockTraceResult) next = start.NumberU64() + 1 + done = make(map[uint64]*blockTraceResult) ) - for res := range results { + for res := range resCh { // Queue up next received result result := &blockTraceResult{ Block: hexutil.Uint64(res.block.NumberU64()), Hash: res.block.Hash(), Traces: res.results, } - // Schedule any parent tries held in memory by this task for dereferencing done[uint64(result.Block)] = result - derefsMu.Lock() - derefTodo = append(derefTodo, res.rootref) - derefsMu.Unlock() - // Stream completed traces to the user, aborting on the first error + + // Stream completed traces to the result channel for result, ok := done[next]; ok; result, ok = done[next] { if len(result.Traces) > 0 || next == end.NumberU64() { - notifier.Notify(sub.ID, result) + // It will be blocked in case the channel consumer doesn't take the + // tracing result in time(e.g. the websocket connect is not stable) + // which will eventually block the entire chain tracer. It's the + // expected behavior to not waste node resources for a non-active user. + retCh <- result } delete(done, next) next++ } } }() - return sub, nil + return retCh } // TraceBlockByNumber returns the structured logs created during the execution of @@ -514,10 +543,12 @@ func (api *API) IntermediateRoots(ctx context.Context, hash common.Hash, config if config != nil && config.Reexec != nil { reexec = *config.Reexec } - statedb, err := api.backend.StateAtBlock(ctx, parent, reexec, nil, true, false) + statedb, release, err := api.backend.StateAtBlock(ctx, parent, reexec, nil, true, false) if err != nil { return nil, err } + defer release() + var ( roots []common.Hash signer = types.MakeSigner(api.backend.ChainConfig(), block.Number()) @@ -562,7 +593,7 @@ func (api *API) StandardTraceBadBlockToFile(ctx context.Context, hash common.Has // traceBlock configures a new tracer according to the provided configuration, and // executes all the transactions contained within. The return value will be one item -// per transaction, dependent on the requestd tracer. +// per transaction, dependent on the requested tracer. func (api *API) traceBlock(ctx context.Context, block *types.Block, config *TraceConfig) ([]*txTraceResult, error) { if block.NumberU64() == 0 { return nil, errors.New("genesis is not traceable") @@ -575,10 +606,12 @@ func (api *API) traceBlock(ctx context.Context, block *types.Block, config *Trac if config != nil && config.Reexec != nil { reexec = *config.Reexec } - statedb, err := api.backend.StateAtBlock(ctx, parent, reexec, nil, true, false) + statedb, release, err := api.backend.StateAtBlock(ctx, parent, reexec, nil, true, false) if err != nil { return nil, err } + defer release() + // Execute all the transaction contained within the block concurrently var ( signer = types.MakeSigner(api.backend.ChainConfig(), block.Number()) @@ -665,10 +698,12 @@ func (api *API) standardTraceBlockToFile(ctx context.Context, block *types.Block if config != nil && config.Reexec != nil { reexec = *config.Reexec } - statedb, err := api.backend.StateAtBlock(ctx, parent, reexec, nil, true, false) + statedb, release, err := api.backend.StateAtBlock(ctx, parent, reexec, nil, true, false) if err != nil { return nil, err } + defer release() + // Retrieve the tracing configurations, or use default values var ( logConfig logger.Config @@ -706,7 +741,7 @@ func (api *API) standardTraceBlockToFile(ctx context.Context, block *types.Block } } for i, tx := range block.Transactions() { - // Prepare the trasaction for un-traced execution + // Prepare the transaction for un-traced execution var ( msg, _ = tx.AsMessage(signer, block.BaseFee()) txContext = core.NewEVMTxContext(msg) @@ -792,10 +827,12 @@ func (api *API) TraceTransaction(ctx context.Context, hash common.Hash, config * if err != nil { return nil, err } - msg, vmctx, statedb, err := api.backend.StateAtTransaction(ctx, block, int(index), reexec) + msg, vmctx, statedb, release, err := api.backend.StateAtTransaction(ctx, block, int(index), reexec) if err != nil { return nil, err } + defer release() + txctx := &Context{ BlockHash: blockHash, TxIndex: int(index), @@ -836,10 +873,12 @@ func (api *API) TraceCall(ctx context.Context, args ethapi.TransactionArgs, bloc if config != nil && config.Reexec != nil { reexec = *config.Reexec } - statedb, err := api.backend.StateAtBlock(ctx, block, reexec, nil, true, false) + statedb, release, err := api.backend.StateAtBlock(ctx, block, reexec, nil, true, false) if err != nil { return nil, err } + defer release() + vmctx := core.NewEVMBlockContext(block.Header(), api.chainContext(ctx), nil) // Apply the customization rules if required. if config != nil { @@ -856,12 +895,7 @@ func (api *API) TraceCall(ctx context.Context, args ethapi.TransactionArgs, bloc var traceConfig *TraceConfig if config != nil { - traceConfig = &TraceConfig{ - Config: config.Config, - Tracer: config.Tracer, - Timeout: config.Timeout, - Reexec: config.Reexec, - } + traceConfig = &config.TraceConfig } return api.traceTx(ctx, msg, new(Context), vmctx, statedb, traceConfig) } @@ -882,7 +916,7 @@ func (api *API) traceTx(ctx context.Context, message core.Message, txctx *Contex // Default tracer is the struct logger tracer = logger.NewStructLogger(config.Config) if config.Tracer != nil { - tracer, err = New(*config.Tracer, txctx) + tracer, err = New(*config.Tracer, txctx, config.TracerConfig) if err != nil { return nil, err } @@ -918,9 +952,7 @@ func APIs(backend Backend) []rpc.API { return []rpc.API{ { Namespace: "debug", - Version: "1.0", Service: NewAPI(backend), - Public: false, }, } } diff --git a/eth/tracers/api_test.go b/eth/tracers/api_test.go index bc12b9275160e..346813ae2c779 100644 --- a/eth/tracers/api_test.go +++ b/eth/tracers/api_test.go @@ -26,6 +26,7 @@ import ( "math/big" "reflect" "sort" + "sync/atomic" "testing" "time" @@ -57,24 +58,21 @@ type testBackend struct { engine consensus.Engine chaindb ethdb.Database chain *core.BlockChain + + refHook func() // Hook is invoked when the requested state is referenced + relHook func() // Hook is invoked when the requested state is released } func newTestBackend(t *testing.T, n int, gspec *core.Genesis, generator func(i int, b *core.BlockGen)) *testBackend { backend := &testBackend{ - chainConfig: params.TestChainConfig, + chainConfig: gspec.Config, engine: ethash.NewFaker(), chaindb: rawdb.NewMemoryDatabase(), } // Generate blocks for testing - gspec.Config = backend.chainConfig - var ( - gendb = rawdb.NewMemoryDatabase() - genesis = gspec.MustCommit(gendb) - ) - blocks, _ := core.GenerateChain(backend.chainConfig, genesis, backend.engine, gendb, n, generator) + _, blocks, _ := core.GenerateChainWithGenesis(gspec, backend.engine, n, generator) // Import the canonical chain - gspec.MustCommit(backend.chaindb) cacheConfig := &core.CacheConfig{ TrieCleanLimit: 256, TrieDirtyLimit: 256, @@ -82,7 +80,7 @@ func newTestBackend(t *testing.T, n int, gspec *core.Genesis, generator func(i i SnapshotLimit: 0, TrieDirtyDisabled: true, // Archive mode } - chain, err := core.NewBlockChain(backend.chaindb, cacheConfig, backend.chainConfig, backend.engine, vm.Config{}, nil, nil) + chain, err := core.NewBlockChain(backend.chaindb, cacheConfig, gspec, nil, backend.engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } @@ -139,25 +137,33 @@ func (b *testBackend) ChainDb() ethdb.Database { return b.chaindb } -func (b *testBackend) StateAtBlock(ctx context.Context, block *types.Block, reexec uint64, base *state.StateDB, checkLive bool, preferDisk bool) (*state.StateDB, error) { +func (b *testBackend) StateAtBlock(ctx context.Context, block *types.Block, reexec uint64, base *state.StateDB, readOnly bool, preferDisk bool) (*state.StateDB, StateReleaseFunc, error) { statedb, err := b.chain.StateAt(block.Root()) if err != nil { - return nil, errStateNotFound + return nil, nil, errStateNotFound + } + if b.refHook != nil { + b.refHook() + } + release := func() { + if b.relHook != nil { + b.relHook() + } } - return statedb, nil + return statedb, release, nil } -func (b *testBackend) StateAtTransaction(ctx context.Context, block *types.Block, txIndex int, reexec uint64) (core.Message, vm.BlockContext, *state.StateDB, error) { +func (b *testBackend) StateAtTransaction(ctx context.Context, block *types.Block, txIndex int, reexec uint64) (core.Message, vm.BlockContext, *state.StateDB, StateReleaseFunc, error) { parent := b.chain.GetBlock(block.ParentHash(), block.NumberU64()-1) if parent == nil { - return nil, vm.BlockContext{}, nil, errBlockNotFound + return nil, vm.BlockContext{}, nil, nil, errBlockNotFound } - statedb, err := b.chain.StateAt(parent.Root()) + statedb, release, err := b.StateAtBlock(ctx, parent, reexec, nil, true, false) if err != nil { - return nil, vm.BlockContext{}, nil, errStateNotFound + return nil, vm.BlockContext{}, nil, nil, errStateNotFound } if txIndex == 0 && len(block.Transactions()) == 0 { - return nil, vm.BlockContext{}, statedb, nil + return nil, vm.BlockContext{}, statedb, release, nil } // Recompute transactions up to the target index. signer := types.MakeSigner(b.chainConfig, block.Number()) @@ -166,15 +172,15 @@ func (b *testBackend) StateAtTransaction(ctx context.Context, block *types.Block txContext := core.NewEVMTxContext(msg) context := core.NewEVMBlockContext(block.Header(), b.chain, nil) if idx == txIndex { - return msg, context, statedb, nil + return msg, context, statedb, release, nil } vmenv := vm.NewEVM(context, txContext, statedb, b.chainConfig, vm.Config{}) if _, err := core.ApplyMessage(vmenv, msg, new(core.GasPool).AddGas(tx.Gas())); err != nil { - return nil, vm.BlockContext{}, nil, fmt.Errorf("transaction %#x failed: %v", tx.Hash(), err) + return nil, vm.BlockContext{}, nil, nil, fmt.Errorf("transaction %#x failed: %v", tx.Hash(), err) } statedb.Finalise(vmenv.ChainConfig().IsEIP158(block.Number())) } - return nil, vm.BlockContext{}, nil, fmt.Errorf("transaction index %d out of range for block %#x", txIndex, block.Hash()) + return nil, vm.BlockContext{}, nil, nil, fmt.Errorf("transaction index %d out of range for block %#x", txIndex, block.Hash()) } func TestTraceCall(t *testing.T) { @@ -182,11 +188,14 @@ func TestTraceCall(t *testing.T) { // Initialize test accounts accounts := newAccounts(3) - genesis := &core.Genesis{Alloc: core.GenesisAlloc{ - accounts[0].addr: {Balance: big.NewInt(params.Ether)}, - accounts[1].addr: {Balance: big.NewInt(params.Ether)}, - accounts[2].addr: {Balance: big.NewInt(params.Ether)}, - }} + genesis := &core.Genesis{ + Config: params.TestChainConfig, + Alloc: core.GenesisAlloc{ + accounts[0].addr: {Balance: big.NewInt(params.Ether)}, + accounts[1].addr: {Balance: big.NewInt(params.Ether)}, + accounts[2].addr: {Balance: big.NewInt(params.Ether)}, + }, + } genBlocks := 10 signer := types.HomesteadSigner{} api := NewAPI(newTestBackend(t, genBlocks, genesis, func(i int, b *core.BlockGen) { @@ -312,10 +321,13 @@ func TestTraceTransaction(t *testing.T) { // Initialize test accounts accounts := newAccounts(2) - genesis := &core.Genesis{Alloc: core.GenesisAlloc{ - accounts[0].addr: {Balance: big.NewInt(params.Ether)}, - accounts[1].addr: {Balance: big.NewInt(params.Ether)}, - }} + genesis := &core.Genesis{ + Config: params.TestChainConfig, + Alloc: core.GenesisAlloc{ + accounts[0].addr: {Balance: big.NewInt(params.Ether)}, + accounts[1].addr: {Balance: big.NewInt(params.Ether)}, + }, + } target := common.Hash{} signer := types.HomesteadSigner{} api := NewAPI(newTestBackend(t, 1, genesis, func(i int, b *core.BlockGen) { @@ -349,11 +361,14 @@ func TestTraceBlock(t *testing.T) { // Initialize test accounts accounts := newAccounts(3) - genesis := &core.Genesis{Alloc: core.GenesisAlloc{ - accounts[0].addr: {Balance: big.NewInt(params.Ether)}, - accounts[1].addr: {Balance: big.NewInt(params.Ether)}, - accounts[2].addr: {Balance: big.NewInt(params.Ether)}, - }} + genesis := &core.Genesis{ + Config: params.TestChainConfig, + Alloc: core.GenesisAlloc{ + accounts[0].addr: {Balance: big.NewInt(params.Ether)}, + accounts[1].addr: {Balance: big.NewInt(params.Ether)}, + accounts[2].addr: {Balance: big.NewInt(params.Ether)}, + }, + } genBlocks := 10 signer := types.HomesteadSigner{} api := NewAPI(newTestBackend(t, genBlocks, genesis, func(i int, b *core.BlockGen) { @@ -424,11 +439,14 @@ func TestTracingWithOverrides(t *testing.T) { t.Parallel() // Initialize test accounts accounts := newAccounts(3) - genesis := &core.Genesis{Alloc: core.GenesisAlloc{ - accounts[0].addr: {Balance: big.NewInt(params.Ether)}, - accounts[1].addr: {Balance: big.NewInt(params.Ether)}, - accounts[2].addr: {Balance: big.NewInt(params.Ether)}, - }} + genesis := &core.Genesis{ + Config: params.TestChainConfig, + Alloc: core.GenesisAlloc{ + accounts[0].addr: {Balance: big.NewInt(params.Ether)}, + accounts[1].addr: {Balance: big.NewInt(params.Ether)}, + accounts[2].addr: {Balance: big.NewInt(params.Ether)}, + }, + } genBlocks := 10 signer := types.HomesteadSigner{} api := NewAPI(newTestBackend(t, genBlocks, genesis, func(i int, b *core.BlockGen) { @@ -616,3 +634,77 @@ func newStates(keys []common.Hash, vals []common.Hash) *map[common.Hash]common.H } return &m } + +func TestTraceChain(t *testing.T) { + // Initialize test accounts + accounts := newAccounts(3) + genesis := &core.Genesis{ + Config: params.TestChainConfig, + Alloc: core.GenesisAlloc{ + accounts[0].addr: {Balance: big.NewInt(params.Ether)}, + accounts[1].addr: {Balance: big.NewInt(params.Ether)}, + accounts[2].addr: {Balance: big.NewInt(params.Ether)}, + }, + } + genBlocks := 50 + signer := types.HomesteadSigner{} + + var ( + ref uint32 // total refs has made + rel uint32 // total rels has made + nonce uint64 + ) + backend := newTestBackend(t, genBlocks, genesis, func(i int, b *core.BlockGen) { + // Transfer from account[0] to account[1] + // value: 1000 wei + // fee: 0 wei + for j := 0; j < i+1; j++ { + tx, _ := types.SignTx(types.NewTransaction(nonce, accounts[1].addr, big.NewInt(1000), params.TxGas, b.BaseFee(), nil), signer, accounts[0].key) + b.AddTx(tx) + nonce += 1 + } + }) + backend.refHook = func() { atomic.AddUint32(&ref, 1) } + backend.relHook = func() { atomic.AddUint32(&rel, 1) } + api := NewAPI(backend) + + single := `{"result":{"gas":21000,"failed":false,"returnValue":"","structLogs":[]}}` + var cases = []struct { + start uint64 + end uint64 + config *TraceConfig + }{ + {0, 50, nil}, // the entire chain range, blocks [1, 50] + {10, 20, nil}, // the middle chain range, blocks [11, 20] + } + for _, c := range cases { + ref, rel = 0, 0 // clean up the counters + + from, _ := api.blockByNumber(context.Background(), rpc.BlockNumber(c.start)) + to, _ := api.blockByNumber(context.Background(), rpc.BlockNumber(c.end)) + resCh := api.traceChain(from, to, c.config, nil) + + next := c.start + 1 + for result := range resCh { + if next != uint64(result.Block) { + t.Error("Unexpected tracing block") + } + if len(result.Traces) != int(next) { + t.Error("Unexpected tracing result") + } + for _, trace := range result.Traces { + blob, _ := json.Marshal(trace) + if string(blob) != single { + t.Error("Unexpected tracing result") + } + } + next += 1 + } + if next != c.end+1 { + t.Error("Missing tracing block") + } + if ref != rel { + t.Errorf("Ref and deref actions are not equal, ref %d rel %d", ref, rel) + } + } +} diff --git a/eth/tracers/internal/tracetest/calltrace_test.go b/eth/tracers/internal/tracetest/calltrace_test.go index cbf20ed00c0ce..90f25e65bb7e5 100644 --- a/eth/tracers/internal/tracetest/calltrace_test.go +++ b/eth/tracers/internal/tracetest/calltrace_test.go @@ -39,7 +39,7 @@ import ( "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/tests" - // Force-load native and js pacakges, to trigger registration + // Force-load native and js packages, to trigger registration _ "github.com/ethereum/go-ethereum/eth/tracers/js" _ "github.com/ethereum/go-ethereum/eth/tracers/native" ) @@ -104,24 +104,26 @@ type callContext struct { // callTrace is the result of a callTracer run. type callTrace struct { - Type string `json:"type"` - From common.Address `json:"from"` - To common.Address `json:"to"` - Input hexutil.Bytes `json:"input"` - Output hexutil.Bytes `json:"output"` - Gas *hexutil.Uint64 `json:"gas,omitempty"` - GasUsed *hexutil.Uint64 `json:"gasUsed,omitempty"` - Value *hexutil.Big `json:"value,omitempty"` - Error string `json:"error,omitempty"` - Calls []callTrace `json:"calls,omitempty"` + Type string `json:"type"` + From common.Address `json:"from"` + To common.Address `json:"to"` + Input hexutil.Bytes `json:"input"` + Output hexutil.Bytes `json:"output"` + Gas *hexutil.Uint64 `json:"gas,omitempty"` + GasUsed *hexutil.Uint64 `json:"gasUsed,omitempty"` + Value *hexutil.Big `json:"value,omitempty"` + Error string `json:"error,omitempty"` + Revertal string `json:"revertReason,omitempty"` + Calls []callTrace `json:"calls,omitempty"` } // callTracerTest defines a single test to check the call tracer against. type callTracerTest struct { - Genesis *core.Genesis `json:"genesis"` - Context *callContext `json:"context"` - Input string `json:"input"` - Result *callTrace `json:"result"` + Genesis *core.Genesis `json:"genesis"` + Context *callContext `json:"context"` + Input string `json:"input"` + TracerConfig json.RawMessage `json:"tracerConfig"` + Result *callTrace `json:"result"` } // Iterates over all the input-output datasets in the tracer test harness and @@ -179,7 +181,7 @@ func testCallTracer(tracerName string, dirPath string, t *testing.T) { } _, statedb = tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false) ) - tracer, err := tracers.New(tracerName, new(tracers.Context)) + tracer, err := tracers.New(tracerName, new(tracers.Context), test.TracerConfig) if err != nil { t.Fatalf("failed to create call tracer: %v", err) } @@ -293,7 +295,7 @@ func benchTracer(tracerName string, test *callTracerTest, b *testing.B) { b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { - tracer, err := tracers.New(tracerName, new(tracers.Context)) + tracer, err := tracers.New(tracerName, new(tracers.Context), nil) if err != nil { b.Fatalf("failed to create call tracer: %v", err) } @@ -359,7 +361,7 @@ func TestZeroValueToNotExitCall(t *testing.T) { } _, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), alloc, false) // Create the tracer, the EVM environment and run it - tracer, err := tracers.New("callTracer", nil) + tracer, err := tracers.New("callTracer", nil, nil) if err != nil { t.Fatalf("failed to create call tracer: %v", err) } diff --git a/eth/tracers/internal/tracetest/testdata/call_tracer/revert_reason.json b/eth/tracers/internal/tracetest/testdata/call_tracer/revert_reason.json index 094b0446779fe..e0b2a9c6f1812 100644 --- a/eth/tracers/internal/tracetest/testdata/call_tracer/revert_reason.json +++ b/eth/tracers/internal/tracetest/testdata/call_tracer/revert_reason.json @@ -59,6 +59,7 @@ "to": "0xf58833cf0c791881b494eb79d461e08a1f043f52", "type": "CALL", "value": "0x0", - "output": "0x08c379a00000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000001e53656c662d64656c65676174696f6e20697320646973616c6c6f7765642e0000" + "output": "0x08c379a00000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000001e53656c662d64656c65676174696f6e20697320646973616c6c6f7765642e0000", + "revertReason": "Self-delegation is disallowed." } } diff --git a/eth/tracers/internal/tracetest/testdata/call_tracer/simple_onlytop.json b/eth/tracers/internal/tracetest/testdata/call_tracer/simple_onlytop.json new file mode 100644 index 0000000000000..ac1fef44098e7 --- /dev/null +++ b/eth/tracers/internal/tracetest/testdata/call_tracer/simple_onlytop.json @@ -0,0 +1,72 @@ +{ + "context": { + "difficulty": "3502894804", + "gasLimit": "4722976", + "miner": "0x1585936b53834b021f68cc13eeefdec2efc8e724", + "number": "2289806", + "timestamp": "1513601314" + }, + "genesis": { + "alloc": { + "0x0024f658a46fbb89d8ac105e98d7ac7cbbaf27c5": { + "balance": "0x0", + "code": "0x", + "nonce": "22", + "storage": {} + }, + "0x3b873a919aa0512d5a0f09e6dcceaa4a6727fafe": { + "balance": "0x4d87094125a369d9bd5", + "code": "0x606060405236156100935763ffffffff60e060020a60003504166311ee8382811461009c57806313af4035146100be5780631f5e8f4c146100ee57806324daddc5146101125780634921a91a1461013b57806363e4bff414610157578063764978f91461017f578063893d20e8146101a1578063ba40aaa1146101cd578063cebc9a82146101f4578063e177246e14610216575b61009a5b5b565b005b34156100a457fe5b6100ac61023d565b60408051918252519081900360200190f35b34156100c657fe5b6100da600160a060020a0360043516610244565b604080519115158252519081900360200190f35b34156100f657fe5b6100da610307565b604080519115158252519081900360200190f35b341561011a57fe5b6100da6004351515610318565b604080519115158252519081900360200190f35b6100da6103d6565b604080519115158252519081900360200190f35b6100da600160a060020a0360043516610420565b604080519115158252519081900360200190f35b341561018757fe5b6100ac61046c565b60408051918252519081900360200190f35b34156101a957fe5b6101b1610473565b60408051600160a060020a039092168252519081900360200190f35b34156101d557fe5b6100da600435610483565b604080519115158252519081900360200190f35b34156101fc57fe5b6100ac61050d565b60408051918252519081900360200190f35b341561021e57fe5b6100da600435610514565b604080519115158252519081900360200190f35b6003545b90565b60006000610250610473565b600160a060020a031633600160a060020a03161415156102705760006000fd5b600160a060020a03831615156102865760006000fd5b50600054600160a060020a0390811690831681146102fb57604051600160a060020a0380851691908316907ffcf23a92150d56e85e3a3d33b357493246e55783095eb6a733eb8439ffc752c890600090a360008054600160a060020a031916600160a060020a03851617905560019150610300565b600091505b5b50919050565b60005460a060020a900460ff165b90565b60006000610324610473565b600160a060020a031633600160a060020a03161415156103445760006000fd5b5060005460a060020a900460ff16801515831515146102fb576000546040805160a060020a90920460ff1615158252841515602083015280517fe6cd46a119083b86efc6884b970bfa30c1708f53ba57b86716f15b2f4551a9539281900390910190a16000805460a060020a60ff02191660a060020a8515150217905560019150610300565b600091505b5b50919050565b60006103e0610307565b801561040557506103ef610473565b600160a060020a031633600160a060020a031614155b156104105760006000fd5b610419336105a0565b90505b5b90565b600061042a610307565b801561044f5750610439610473565b600160a060020a031633600160a060020a031614155b1561045a5760006000fd5b610463826105a0565b90505b5b919050565b6001545b90565b600054600160a060020a03165b90565b6000600061048f610473565b600160a060020a031633600160a060020a03161415156104af5760006000fd5b506001548281146102fb57604080518281526020810185905281517f79a3746dde45672c9e8ab3644b8bb9c399a103da2dc94b56ba09777330a83509929181900390910190a160018381559150610300565b600091505b5b50919050565b6002545b90565b60006000610520610473565b600160a060020a031633600160a060020a03161415156105405760006000fd5b506002548281146102fb57604080518281526020810185905281517ff6991a728965fedd6e927fdf16bdad42d8995970b4b31b8a2bf88767516e2494929181900390910190a1600283905560019150610300565b600091505b5b50919050565b60006000426105ad61023d565b116102fb576105c46105bd61050d565b4201610652565b6105cc61046c565b604051909150600160a060020a038416908290600081818185876187965a03f1925050501561063d57604080518281529051600160a060020a038516917f9bca65ce52fdef8a470977b51f247a2295123a4807dfa9e502edf0d30722da3b919081900360200190a260019150610300565b6102fb42610652565b5b600091505b50919050565b60038190555b505600a165627a7a72305820f3c973c8b7ed1f62000b6701bd5b708469e19d0f1d73fde378a56c07fd0b19090029", + "nonce": "1", + "storage": { + "0x0000000000000000000000000000000000000000000000000000000000000000": "0x000000000000000000000001b436ba50d378d4bbc8660d312a13df6af6e89dfb", + "0x0000000000000000000000000000000000000000000000000000000000000001": "0x00000000000000000000000000000000000000000000000006f05b59d3b20000", + "0x0000000000000000000000000000000000000000000000000000000000000002": "0x000000000000000000000000000000000000000000000000000000000000003c", + "0x0000000000000000000000000000000000000000000000000000000000000003": "0x000000000000000000000000000000000000000000000000000000005a37b834" + } + }, + "0xb436ba50d378d4bbc8660d312a13df6af6e89dfb": { + "balance": "0x1780d77678137ac1b775", + "code": "0x", + "nonce": "29072", + "storage": {} + } + }, + "config": { + "byzantiumBlock": 1700000, + "chainId": 3, + "daoForkSupport": true, + "eip150Block": 0, + "eip150Hash": "0x41941023680923e0fe4d74a34bdac8141f2540e3ae90623718e47d66d1ca4a2d", + "eip155Block": 10, + "eip158Block": 10, + "ethash": {}, + "homesteadBlock": 0 + }, + "difficulty": "3509749784", + "extraData": "0x4554482e45544846414e532e4f52472d4641313738394444", + "gasLimit": "4727564", + "hash": "0x609948ac3bd3c00b7736b933248891d6c901ee28f066241bddb28f4e00a9f440", + "miner": "0xbbf5029fd710d227630c8b7d338051b8e76d50b3", + "mixHash": "0xb131e4507c93c7377de00e7c271bf409ec7492767142ff0f45c882f8068c2ada", + "nonce": "0x4eb12e19c16d43da", + "number": "2289805", + "stateRoot": "0xc7f10f352bff82fac3c2999d3085093d12652e19c7fd32591de49dc5d91b4f1f", + "timestamp": "1513601261", + "totalDifficulty": "7143276353481064" + }, + "input": "0xf88b8271908506fc23ac0083015f90943b873a919aa0512d5a0f09e6dcceaa4a6727fafe80a463e4bff40000000000000000000000000024f658a46fbb89d8ac105e98d7ac7cbbaf27c52aa0bdce0b59e8761854e857fe64015f06dd08a4fbb7624f6094893a79a72e6ad6bea01d9dde033cff7bb235a3163f348a6d7ab8d6b52bc0963a95b91612e40ca766a4", + "tracerConfig": { + "onlyTopCall": true + }, + "result": { + "from": "0xb436ba50d378d4bbc8660d312a13df6af6e89dfb", + "gas": "0x10738", + "gasUsed": "0x3ef9", + "input": "0x63e4bff40000000000000000000000000024f658a46fbb89d8ac105e98d7ac7cbbaf27c5", + "output": "0x0000000000000000000000000000000000000000000000000000000000000001", + "to": "0x3b873a919aa0512d5a0f09e6dcceaa4a6727fafe", + "type": "CALL", + "value": "0x0" + } +} diff --git a/eth/tracers/js/goja.go b/eth/tracers/js/goja.go index f0c78c084bd95..7bb323f6985ce 100644 --- a/eth/tracers/js/goja.go +++ b/eth/tracers/js/goja.go @@ -33,6 +33,10 @@ import ( jsassets "github.com/ethereum/go-ethereum/eth/tracers/js/internal/tracers" ) +const ( + memoryPadLimit = 1024 * 1024 +) + var assetTracers = make(map[string]string) // init retrieves the JavaScript transaction tracers included in go-ethereum. @@ -55,11 +59,7 @@ type fromBufFn = func(vm *goja.Runtime, buf goja.Value, allowString bool) ([]byt func toBuf(vm *goja.Runtime, bufType goja.Value, val []byte) (goja.Value, error) { // bufType is usually Uint8Array. This is equivalent to `new Uint8Array(val)` in JS. - res, err := vm.New(bufType, vm.ToValue(val)) - if err != nil { - return nil, err - } - return vm.ToValue(res), nil + return vm.New(bufType, vm.ToValue(vm.NewArrayBuffer(val))) } func fromBuf(vm *goja.Runtime, bufType goja.Value, buf goja.Value, allowString bool) ([]byte, error) { @@ -70,6 +70,7 @@ func fromBuf(vm *goja.Runtime, bufType goja.Value, buf goja.Value, allowString b break } return common.FromHex(obj.String()), nil + case "Array": var b []byte if err := vm.ExportTo(buf, &b); err != nil { @@ -81,10 +82,7 @@ func fromBuf(vm *goja.Runtime, bufType goja.Value, buf goja.Value, allowString b if !obj.Get("constructor").SameAs(bufType) { break } - var b []byte - if err := vm.ExportTo(buf, &b); err != nil { - return nil, err - } + b := obj.Get("buffer").Export().(goja.ArrayBuffer).Bytes() return b, nil } return nil, fmt.Errorf("invalid buffer type") @@ -131,7 +129,7 @@ type jsTracer struct { // The methods `result` and `fault` are required to be present. // The methods `step`, `enter`, and `exit` are optional, but note that // `enter` and `exit` always go together. -func newJsTracer(code string, ctx *tracers.Context) (tracers.Tracer, error) { +func newJsTracer(code string, ctx *tracers.Context, cfg json.RawMessage) (tracers.Tracer, error) { if c, ok := assetTracers[code]; ok { code = c } @@ -183,6 +181,17 @@ func newJsTracer(code string, ctx *tracers.Context) (tracers.Tracer, error) { t.exit = exit t.result = result t.fault = fault + + // Pass in config + if setup, ok := goja.AssertFunction(obj.Get("setup")); ok { + cfgStr := "{}" + if cfg != nil { + cfgStr = string(cfg) + } + if _, err := setup(obj, vm.ToValue(cfgStr)); err != nil { + return nil, err + } + } // Setup objects carrying data to JS. These are created once and re-used. t.log = &steplog{ vm: vm, @@ -251,10 +260,11 @@ func (t *jsTracer) CaptureState(pc uint64, op vm.OpCode, gas, cost uint64, scope log.memory.memory = scope.Memory log.stack.stack = scope.Stack log.contract.contract = scope.Contract - log.pc = uint(pc) - log.gas = uint(gas) - log.cost = uint(cost) - log.depth = uint(depth) + log.pc = pc + log.gas = gas + log.cost = cost + log.refund = t.env.StateDB.GetRefund() + log.depth = depth log.err = err if _, err := t.step(t.obj, t.logValue, t.dbValue); err != nil { t.onError("step", err) @@ -554,12 +564,17 @@ func (mo *memoryObj) slice(begin, end int64) ([]byte, error) { return []byte{}, nil } if end < begin || begin < 0 { - return nil, fmt.Errorf("Tracer accessed out of bound memory: offset %d, end %d", begin, end) + return nil, fmt.Errorf("tracer accessed out of bound memory: offset %d, end %d", begin, end) } - if mo.memory.Len() < int(end) { - return nil, fmt.Errorf("Tracer accessed out of bound memory: available %d, offset %d, size %d", mo.memory.Len(), begin, end-begin) + mlen := mo.memory.Len() + if end-int64(mlen) > memoryPadLimit { + return nil, fmt.Errorf("tracer reached limit for padding memory slice: end %d, memorySize %d", end, mlen) } - return mo.memory.GetCopy(begin, end-begin), nil + slice := make([]byte, end-begin) + end = min(end, int64(mo.memory.Len())) + ptr := mo.memory.GetPtr(begin, end-begin) + copy(slice[:], ptr[:]) + return slice, nil } func (mo *memoryObj) GetUint(addr int64) goja.Value { @@ -579,7 +594,7 @@ func (mo *memoryObj) GetUint(addr int64) goja.Value { // getUint returns the 32 bytes at the specified address interpreted as a uint. func (mo *memoryObj) getUint(addr int64) (*big.Int, error) { if mo.memory.Len() < int(addr)+32 || addr < 0 { - return nil, fmt.Errorf("Tracer accessed out of bound memory: available %d, offset %d, size %d", mo.memory.Len(), addr, 32) + return nil, fmt.Errorf("tracer accessed out of bound memory: available %d, offset %d, size %d", mo.memory.Len(), addr, 32) } return new(big.Int).SetBytes(mo.memory.GetPtr(addr, 32)), nil } @@ -619,7 +634,7 @@ func (s *stackObj) Peek(idx int) goja.Value { // peek returns the nth-from-the-top element of the stack. func (s *stackObj) peek(idx int) (*big.Int, error) { if len(s.stack.Data()) <= idx || idx < 0 { - return nil, fmt.Errorf("Tracer accessed out of bound stack: size %d, index %d", len(s.stack.Data()), idx) + return nil, fmt.Errorf("tracer accessed out of bound stack: size %d, index %d", len(s.stack.Data()), idx) } return s.stack.Back(idx).ToBig(), nil } @@ -765,7 +780,7 @@ func (co *contractObj) GetValue() goja.Value { } func (co *contractObj) GetInput() goja.Value { - input := co.contract.Input + input := common.CopyBytes(co.contract.Input) res, err := co.toBuf(co.vm, input) if err != nil { co.vm.Interrupt(err) @@ -884,7 +899,6 @@ func (r *callframeResult) GetError() goja.Value { return r.vm.ToValue(r.err.Error()) } return goja.Undefined() - } func (r *callframeResult) setupObject() *goja.Object { @@ -903,33 +917,19 @@ type steplog struct { stack *stackObj contract *contractObj - pc uint - gas uint - cost uint - depth uint - refund uint + pc uint64 + gas uint64 + cost uint64 + depth int + refund uint64 err error } -func (l *steplog) GetPC() uint { - return l.pc -} - -func (l *steplog) GetGas() uint { - return l.gas -} - -func (l *steplog) GetCost() uint { - return l.cost -} - -func (l *steplog) GetDepth() uint { - return l.depth -} - -func (l *steplog) GetRefund() uint { - return l.refund -} +func (l *steplog) GetPC() uint64 { return l.pc } +func (l *steplog) GetGas() uint64 { return l.gas } +func (l *steplog) GetCost() uint64 { return l.cost } +func (l *steplog) GetDepth() int { return l.depth } +func (l *steplog) GetRefund() uint64 { return l.refund } func (l *steplog) GetError() goja.Value { if l.err != nil { @@ -954,3 +954,10 @@ func (l *steplog) setupObject() *goja.Object { o.Set("contract", l.contract.setupObject()) return o } + +func min(a, b int64) int64 { + if a < b { + return a + } + return b +} diff --git a/eth/tracers/js/internal/tracers/4byte_tracer_legacy.js b/eth/tracers/js/internal/tracers/4byte_tracer_legacy.js index 462b4ad4cb550..e4714b8bfb762 100644 --- a/eth/tracers/js/internal/tracers/4byte_tracer_legacy.js +++ b/eth/tracers/js/internal/tracers/4byte_tracer_legacy.js @@ -46,7 +46,7 @@ return false; }, - // store save the given indentifier and datasize. + // store save the given identifier and datasize. store: function(id, size){ var key = "" + toHex(id) + "-" + size; this.ids[key] = this.ids[key] + 1 || 1; diff --git a/eth/tracers/js/tracer_test.go b/eth/tracers/js/tracer_test.go index 2863bd4451b8a..02789d6713ed9 100644 --- a/eth/tracers/js/tracer_test.go +++ b/eth/tracers/js/tracer_test.go @@ -60,7 +60,7 @@ func testCtx() *vmContext { return &vmContext{blockCtx: vm.BlockContext{BlockNumber: big.NewInt(1)}, txCtx: vm.TxContext{GasPrice: big.NewInt(100000)}} } -func runTrace(tracer tracers.Tracer, vmctx *vmContext, chaincfg *params.ChainConfig) (json.RawMessage, error) { +func runTrace(tracer tracers.Tracer, vmctx *vmContext, chaincfg *params.ChainConfig, contractCode []byte) (json.RawMessage, error) { var ( env = vm.NewEVM(vmctx.blockCtx, vmctx.txCtx, &dummyStatedb{}, chaincfg, vm.Config{Debug: true, Tracer: tracer}) gasLimit uint64 = 31000 @@ -69,6 +69,9 @@ func runTrace(tracer tracers.Tracer, vmctx *vmContext, chaincfg *params.ChainCon contract = vm.NewContract(account{}, account{}, value, startGas) ) contract.Code = []byte{byte(vm.PUSH1), 0x1, byte(vm.PUSH1), 0x1, 0x0} + if contractCode != nil { + contract.Code = contractCode + } tracer.CaptureTxStart(gasLimit) tracer.CaptureStart(env, contract.Caller(), contract.Address(), false, []byte{}, startGas, value) @@ -83,35 +86,36 @@ func runTrace(tracer tracers.Tracer, vmctx *vmContext, chaincfg *params.ChainCon } func TestTracer(t *testing.T) { - execTracer := func(code string) ([]byte, string) { + execTracer := func(code string, contract []byte) ([]byte, string) { t.Helper() - tracer, err := newJsTracer(code, nil) + tracer, err := newJsTracer(code, nil, nil) if err != nil { t.Fatal(err) } - ret, err := runTrace(tracer, testCtx(), params.TestChainConfig) + ret, err := runTrace(tracer, testCtx(), params.TestChainConfig, contract) if err != nil { return nil, err.Error() // Stringify to allow comparison without nil checks } return ret, "" } for i, tt := range []struct { - code string - want string - fail string + code string + want string + fail string + contract []byte }{ { // tests that we don't panic on bad arguments to memory access code: "{depths: [], step: function(log) { this.depths.push(log.memory.slice(-1,-2)); }, fault: function() {}, result: function() { return this.depths; }}", want: ``, - fail: "Tracer accessed out of bound memory: offset -1, end -2 at step (:1:53(15)) in server-side tracer function 'step'", + fail: "tracer accessed out of bound memory: offset -1, end -2 at step (:1:53(15)) in server-side tracer function 'step'", }, { // tests that we don't panic on bad arguments to stack peeks code: "{depths: [], step: function(log) { this.depths.push(log.stack.peek(-1)); }, fault: function() {}, result: function() { return this.depths; }}", want: ``, - fail: "Tracer accessed out of bound stack: size 0, index -1 at step (:1:53(13)) in server-side tracer function 'step'", + fail: "tracer accessed out of bound stack: size 0, index -1 at step (:1:53(13)) in server-side tracer function 'step'", }, { // tests that we don't panic on bad arguments to memory getUint code: "{ depths: [], step: function(log, db) { this.depths.push(log.memory.getUint(-64));}, fault: function() {}, result: function() { return this.depths; }}", want: ``, - fail: "Tracer accessed out of bound memory: available 0, offset -64, size 32 at step (:1:58(13)) in server-side tracer function 'step'", + fail: "tracer accessed out of bound memory: available 0, offset -64, size 32 at step (:1:58(13)) in server-side tracer function 'step'", }, { // tests some general counting code: "{count: 0, step: function() { this.count += 1; }, fault: function() {}, result: function() { return this.count; }}", want: `3`, @@ -139,9 +143,18 @@ func TestTracer(t *testing.T) { }, { code: "{res: null, step: function(log) { var address = Array.prototype.slice.call(log.contract.getAddress()); this.res = toAddress(address); }, fault: function() {}, result: function() { return this.res }}", want: `{"0":0,"1":0,"2":0,"3":0,"4":0,"5":0,"6":0,"7":0,"8":0,"9":0,"10":0,"11":0,"12":0,"13":0,"14":0,"15":0,"16":0,"17":0,"18":0,"19":0}`, + }, { + code: "{res: [], step: function(log) { var op = log.op.toString(); if (op === 'MSTORE8' || op === 'STOP') { this.res.push(log.memory.slice(0, 2)) } }, fault: function() {}, result: function() { return this.res }}", + want: `[{"0":0,"1":0},{"0":255,"1":0}]`, + contract: []byte{byte(vm.PUSH1), byte(0xff), byte(vm.PUSH1), byte(0x00), byte(vm.MSTORE8), byte(vm.STOP)}, + }, { + code: "{res: [], step: function(log) { if (log.op.toString() === 'STOP') { this.res.push(log.memory.slice(5, 1025 * 1024)) } }, fault: function() {}, result: function() { return this.res }}", + want: "", + fail: "tracer reached limit for padding memory slice: end 1049600, memorySize 32 at step (:1:83(23)) in server-side tracer function 'step'", + contract: []byte{byte(vm.PUSH1), byte(0xff), byte(vm.PUSH1), byte(0x00), byte(vm.MSTORE8), byte(vm.STOP)}, }, } { - if have, err := execTracer(tt.code); tt.want != string(have) || tt.fail != err { + if have, err := execTracer(tt.code, tt.contract); tt.want != string(have) || tt.fail != err { t.Errorf("testcase %d: expected return value to be '%s' got '%s', error to be '%s' got '%s'\n\tcode: %v", i, tt.want, string(have), tt.fail, err, tt.code) } } @@ -149,7 +162,7 @@ func TestTracer(t *testing.T) { func TestHalt(t *testing.T) { timeout := errors.New("stahp") - tracer, err := newJsTracer("{step: function() { while(1); }, result: function() { return null; }, fault: function(){}}", nil) + tracer, err := newJsTracer("{step: function() { while(1); }, result: function() { return null; }, fault: function(){}}", nil, nil) if err != nil { t.Fatal(err) } @@ -157,13 +170,13 @@ func TestHalt(t *testing.T) { time.Sleep(1 * time.Second) tracer.Stop(timeout) }() - if _, err = runTrace(tracer, testCtx(), params.TestChainConfig); !strings.Contains(err.Error(), "stahp") { + if _, err = runTrace(tracer, testCtx(), params.TestChainConfig, nil); !strings.Contains(err.Error(), "stahp") { t.Errorf("Expected timeout error, got %v", err) } } func TestHaltBetweenSteps(t *testing.T) { - tracer, err := newJsTracer("{step: function() {}, fault: function() {}, result: function() { return null; }}", nil) + tracer, err := newJsTracer("{step: function() {}, fault: function() {}, result: function() { return null; }}", nil, nil) if err != nil { t.Fatal(err) } @@ -187,7 +200,7 @@ func TestHaltBetweenSteps(t *testing.T) { func TestNoStepExec(t *testing.T) { execTracer := func(code string) []byte { t.Helper() - tracer, err := newJsTracer(code, nil) + tracer, err := newJsTracer(code, nil, nil) if err != nil { t.Fatal(err) } @@ -221,41 +234,41 @@ func TestIsPrecompile(t *testing.T) { chaincfg.IstanbulBlock = big.NewInt(200) chaincfg.BerlinBlock = big.NewInt(300) txCtx := vm.TxContext{GasPrice: big.NewInt(100000)} - tracer, err := newJsTracer("{addr: toAddress('0000000000000000000000000000000000000009'), res: null, step: function() { this.res = isPrecompiled(this.addr); }, fault: function() {}, result: function() { return this.res; }}", nil) + tracer, err := newJsTracer("{addr: toAddress('0000000000000000000000000000000000000009'), res: null, step: function() { this.res = isPrecompiled(this.addr); }, fault: function() {}, result: function() { return this.res; }}", nil, nil) if err != nil { t.Fatal(err) } blockCtx := vm.BlockContext{BlockNumber: big.NewInt(150)} - res, err := runTrace(tracer, &vmContext{blockCtx, txCtx}, chaincfg) + res, err := runTrace(tracer, &vmContext{blockCtx, txCtx}, chaincfg, nil) if err != nil { t.Error(err) } if string(res) != "false" { - t.Errorf("Tracer should not consider blake2f as precompile in byzantium") + t.Errorf("tracer should not consider blake2f as precompile in byzantium") } - tracer, _ = newJsTracer("{addr: toAddress('0000000000000000000000000000000000000009'), res: null, step: function() { this.res = isPrecompiled(this.addr); }, fault: function() {}, result: function() { return this.res; }}", nil) + tracer, _ = newJsTracer("{addr: toAddress('0000000000000000000000000000000000000009'), res: null, step: function() { this.res = isPrecompiled(this.addr); }, fault: function() {}, result: function() { return this.res; }}", nil, nil) blockCtx = vm.BlockContext{BlockNumber: big.NewInt(250)} - res, err = runTrace(tracer, &vmContext{blockCtx, txCtx}, chaincfg) + res, err = runTrace(tracer, &vmContext{blockCtx, txCtx}, chaincfg, nil) if err != nil { t.Error(err) } if string(res) != "true" { - t.Errorf("Tracer should consider blake2f as precompile in istanbul") + t.Errorf("tracer should consider blake2f as precompile in istanbul") } } func TestEnterExit(t *testing.T) { // test that either both or none of enter() and exit() are defined - if _, err := newJsTracer("{step: function() {}, fault: function() {}, result: function() { return null; }, enter: function() {}}", new(tracers.Context)); err == nil { + if _, err := newJsTracer("{step: function() {}, fault: function() {}, result: function() { return null; }, enter: function() {}}", new(tracers.Context), nil); err == nil { t.Fatal("tracer creation should've failed without exit() definition") } - if _, err := newJsTracer("{step: function() {}, fault: function() {}, result: function() { return null; }, enter: function() {}, exit: function() {}}", new(tracers.Context)); err != nil { + if _, err := newJsTracer("{step: function() {}, fault: function() {}, result: function() { return null; }, enter: function() {}, exit: function() {}}", new(tracers.Context), nil); err != nil { t.Fatal(err) } // test that the enter and exit method are correctly invoked and the values passed - tracer, err := newJsTracer("{enters: 0, exits: 0, enterGas: 0, gasUsed: 0, step: function() {}, fault: function() {}, result: function() { return {enters: this.enters, exits: this.exits, enterGas: this.enterGas, gasUsed: this.gasUsed} }, enter: function(frame) { this.enters++; this.enterGas = frame.getGas(); }, exit: function(res) { this.exits++; this.gasUsed = res.getGasUsed(); }}", new(tracers.Context)) + tracer, err := newJsTracer("{enters: 0, exits: 0, enterGas: 0, gasUsed: 0, step: function() {}, fault: function() {}, result: function() { return {enters: this.enters, exits: this.exits, enterGas: this.enterGas, gasUsed: this.gasUsed} }, enter: function(frame) { this.enters++; this.enterGas = frame.getGas(); }, exit: function(res) { this.exits++; this.gasUsed = res.getGasUsed(); }}", new(tracers.Context), nil) if err != nil { t.Fatal(err) } @@ -274,3 +287,33 @@ func TestEnterExit(t *testing.T) { t.Errorf("Number of invocations of enter() and exit() is wrong. Have %s, want %s\n", have, want) } } + +func TestSetup(t *testing.T) { + // Test empty config + _, err := newJsTracer(`{setup: function(cfg) { if (cfg !== "{}") { throw("invalid empty config") } }, fault: function() {}, result: function() {}}`, new(tracers.Context), nil) + if err != nil { + t.Error(err) + } + + cfg, err := json.Marshal(map[string]string{"foo": "bar"}) + if err != nil { + t.Fatal(err) + } + // Test no setup func + _, err = newJsTracer(`{fault: function() {}, result: function() {}}`, new(tracers.Context), cfg) + if err != nil { + t.Fatal(err) + } + // Test config value + tracer, err := newJsTracer("{config: null, setup: function(cfg) { this.config = JSON.parse(cfg) }, step: function() {}, fault: function() {}, result: function() { return this.config.foo }}", new(tracers.Context), cfg) + if err != nil { + t.Fatal(err) + } + have, err := tracer.GetResult() + if err != nil { + t.Fatal(err) + } + if string(have) != `"bar"` { + t.Errorf("tracer returned wrong result. have: %s, want: \"bar\"\n", string(have)) + } +} diff --git a/eth/tracers/logger/logger.go b/eth/tracers/logger/logger.go index fe850d6b3e61c..07aa2f2b4301c 100644 --- a/eth/tracers/logger/logger.go +++ b/eth/tracers/logger/logger.go @@ -40,7 +40,7 @@ type Storage map[common.Hash]common.Hash // Copy duplicates the current storage. func (s Storage) Copy() Storage { - cpy := make(Storage) + cpy := make(Storage, len(s)) for key, value := range s { cpy[key] = value } @@ -224,7 +224,7 @@ func (l *StructLogger) CaptureEnd(output []byte, gasUsed uint64, t time.Duration l.output = output l.err = err if l.cfg.Debug { - fmt.Printf("0x%x\n", output) + fmt.Printf("%#x\n", output) if err != nil { fmt.Printf(" error: %v\n", err) } @@ -346,11 +346,11 @@ func NewMarkdownLogger(cfg *Config, writer io.Writer) *mdLogger { func (t *mdLogger) CaptureStart(env *vm.EVM, from common.Address, to common.Address, create bool, input []byte, gas uint64, value *big.Int) { t.env = env if !create { - fmt.Fprintf(t.out, "From: `%v`\nTo: `%v`\nData: `0x%x`\nGas: `%d`\nValue `%v` wei\n", + fmt.Fprintf(t.out, "From: `%v`\nTo: `%v`\nData: `%#x`\nGas: `%d`\nValue `%v` wei\n", from.String(), to.String(), input, gas, value) } else { - fmt.Fprintf(t.out, "From: `%v`\nCreate at: `%v`\nData: `0x%x`\nGas: `%d`\nValue `%v` wei\n", + fmt.Fprintf(t.out, "From: `%v`\nCreate at: `%v`\nData: `%#x`\nGas: `%d`\nValue `%v` wei\n", from.String(), to.String(), input, gas, value) } @@ -387,7 +387,7 @@ func (t *mdLogger) CaptureFault(pc uint64, op vm.OpCode, gas, cost uint64, scope } func (t *mdLogger) CaptureEnd(output []byte, gasUsed uint64, tm time.Duration, err error) { - fmt.Fprintf(t.out, "\nOutput: `0x%x`\nConsumed gas: `%d`\nError: `%v`\n", + fmt.Fprintf(t.out, "\nOutput: `%#x`\nConsumed gas: `%d`\nError: `%v`\n", output, gasUsed, err) } diff --git a/eth/tracers/native/4byte.go b/eth/tracers/native/4byte.go index 92cc70994c323..29f3bccb1a56c 100644 --- a/eth/tracers/native/4byte.go +++ b/eth/tracers/native/4byte.go @@ -37,14 +37,15 @@ func init() { // a reversed signature can be matched against the size of the data. // // Example: -// > debug.traceTransaction( "0x214e597e35da083692f5386141e69f47e973b2c56e7a8073b1ea08fd7571e9de", {tracer: "4byteTracer"}) -// { -// 0x27dc297e-128: 1, -// 0x38cc4831-0: 2, -// 0x524f3889-96: 1, -// 0xadf59f99-288: 1, -// 0xc281d19e-0: 1 -// } +// +// > debug.traceTransaction( "0x214e597e35da083692f5386141e69f47e973b2c56e7a8073b1ea08fd7571e9de", {tracer: "4byteTracer"}) +// { +// 0x27dc297e-128: 1, +// 0x38cc4831-0: 2, +// 0x524f3889-96: 1, +// 0xadf59f99-288: 1, +// 0xc281d19e-0: 1 +// } type fourByteTracer struct { env *vm.EVM ids map[string]int // ids aggregates the 4byte ids found @@ -55,11 +56,11 @@ type fourByteTracer struct { // newFourByteTracer returns a native go tracer which collects // 4 byte-identifiers of a tx, and implements vm.EVMLogger. -func newFourByteTracer(ctx *tracers.Context) tracers.Tracer { +func newFourByteTracer(ctx *tracers.Context, _ json.RawMessage) (tracers.Tracer, error) { t := &fourByteTracer{ ids: make(map[string]int), } - return t + return t, nil } // isPrecompiled returns whether the addr is a precompile. Logic borrowed from newJsTracer in eth/tracers/js/tracer.go @@ -150,3 +151,7 @@ func (t *fourByteTracer) Stop(err error) { t.reason = err atomic.StoreUint32(&t.interrupt, 1) } + +func bytesToHex(s []byte) string { + return "0x" + common.Bytes2Hex(s) +} diff --git a/eth/tracers/native/call.go b/eth/tracers/native/call.go index d334e328a5ffb..26fdd0008d2ae 100644 --- a/eth/tracers/native/call.go +++ b/eth/tracers/native/call.go @@ -20,74 +20,111 @@ import ( "encoding/json" "errors" "math/big" - "strconv" - "strings" "sync/atomic" "time" + "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/eth/tracers" ) +//go:generate go run github.com/fjl/gencodec -type callFrame -field-override callFrameMarshaling -out gen_callframe_json.go + func init() { register("callTracer", newCallTracer) } type callFrame struct { - Type string `json:"type"` - From string `json:"from"` - To string `json:"to,omitempty"` - Value string `json:"value,omitempty"` - Gas string `json:"gas"` - GasUsed string `json:"gasUsed"` - Input string `json:"input"` - Output string `json:"output,omitempty"` - Error string `json:"error,omitempty"` - Calls []callFrame `json:"calls,omitempty"` + Type vm.OpCode `json:"-"` + From common.Address `json:"from"` + Gas uint64 `json:"gas"` + GasUsed uint64 `json:"gasUsed"` + To common.Address `json:"to,omitempty" rlp:"optional"` + Input []byte `json:"input" rlp:"optional"` + Output []byte `json:"output,omitempty" rlp:"optional"` + Error string `json:"error,omitempty" rlp:"optional"` + Revertal string `json:"revertReason,omitempty"` + Calls []callFrame `json:"calls,omitempty" rlp:"optional"` + // Placed at end on purpose. The RLP will be decoded to 0 instead of + // nil if there are non-empty elements after in the struct. + Value *big.Int `json:"value,omitempty" rlp:"optional"` +} + +func (f callFrame) TypeString() string { + return f.Type.String() +} + +type callFrameMarshaling struct { + TypeString string `json:"type"` + Gas hexutil.Uint64 + GasUsed hexutil.Uint64 + Value *hexutil.Big + Input hexutil.Bytes + Output hexutil.Bytes } type callTracer struct { env *vm.EVM callstack []callFrame + config callTracerConfig interrupt uint32 // Atomic flag to signal execution interruption reason error // Textual reason for the interruption } +type callTracerConfig struct { + OnlyTopCall bool `json:"onlyTopCall"` // If true, call tracer won't collect any subcalls +} + // newCallTracer returns a native go tracer which tracks // call frames of a tx, and implements vm.EVMLogger. -func newCallTracer(ctx *tracers.Context) tracers.Tracer { +func newCallTracer(ctx *tracers.Context, cfg json.RawMessage) (tracers.Tracer, error) { + var config callTracerConfig + if cfg != nil { + if err := json.Unmarshal(cfg, &config); err != nil { + return nil, err + } + } // First callframe contains tx context info // and is populated on start and end. - return &callTracer{callstack: make([]callFrame, 1)} + return &callTracer{callstack: make([]callFrame, 1), config: config}, nil } // CaptureStart implements the EVMLogger interface to initialize the tracing operation. func (t *callTracer) CaptureStart(env *vm.EVM, from common.Address, to common.Address, create bool, input []byte, gas uint64, value *big.Int) { t.env = env t.callstack[0] = callFrame{ - Type: "CALL", - From: addrToHex(from), - To: addrToHex(to), - Input: bytesToHex(input), - Gas: uintToHex(gas), - Value: bigToHex(value), + Type: vm.CALL, + From: from, + To: to, + Input: common.CopyBytes(input), + Gas: gas, + Value: value, } if create { - t.callstack[0].Type = "CREATE" + t.callstack[0].Type = vm.CREATE } } // CaptureEnd is called after the call finishes to finalize the tracing. func (t *callTracer) CaptureEnd(output []byte, gasUsed uint64, _ time.Duration, err error) { - t.callstack[0].GasUsed = uintToHex(gasUsed) - if err != nil { - t.callstack[0].Error = err.Error() - if err.Error() == "execution reverted" && len(output) > 0 { - t.callstack[0].Output = bytesToHex(output) - } - } else { - t.callstack[0].Output = bytesToHex(output) + t.callstack[0].GasUsed = gasUsed + output = common.CopyBytes(output) + if err == nil { + t.callstack[0].Output = output + return + } + t.callstack[0].Error = err.Error() + if !errors.Is(err, vm.ErrExecutionReverted) || len(output) == 0 { + return + } + t.callstack[0].Output = output + if len(output) < 4 { + return + } + if unpacked, err := abi.UnpackRevert(output); err == nil { + t.callstack[0].Revertal = unpacked } } @@ -101,6 +138,9 @@ func (t *callTracer) CaptureFault(pc uint64, op vm.OpCode, gas, cost uint64, _ * // CaptureEnter is called when EVM enters a new scope (via call, create or selfdestruct). func (t *callTracer) CaptureEnter(typ vm.OpCode, from common.Address, to common.Address, input []byte, gas uint64, value *big.Int) { + if t.config.OnlyTopCall { + return + } // Skip if tracing was interrupted if atomic.LoadUint32(&t.interrupt) > 0 { t.env.Cancel() @@ -108,12 +148,12 @@ func (t *callTracer) CaptureEnter(typ vm.OpCode, from common.Address, to common. } call := callFrame{ - Type: typ.String(), - From: addrToHex(from), - To: addrToHex(to), - Input: bytesToHex(input), - Gas: uintToHex(gas), - Value: bigToHex(value), + Type: typ, + From: from, + To: to, + Input: common.CopyBytes(input), + Gas: gas, + Value: value, } t.callstack = append(t.callstack, call) } @@ -121,6 +161,9 @@ func (t *callTracer) CaptureEnter(typ vm.OpCode, from common.Address, to common. // CaptureExit is called when EVM exits a scope, even if the scope didn't // execute any code. func (t *callTracer) CaptureExit(output []byte, gasUsed uint64, err error) { + if t.config.OnlyTopCall { + return + } size := len(t.callstack) if size <= 1 { return @@ -130,13 +173,13 @@ func (t *callTracer) CaptureExit(output []byte, gasUsed uint64, err error) { t.callstack = t.callstack[:size-1] size -= 1 - call.GasUsed = uintToHex(gasUsed) + call.GasUsed = gasUsed if err == nil { - call.Output = bytesToHex(output) + call.Output = common.CopyBytes(output) } else { call.Error = err.Error() - if call.Type == "CREATE" || call.Type == "CREATE2" { - call.To = "" + if call.Type == vm.CREATE || call.Type == vm.CREATE2 { + call.To = common.Address{} } } t.callstack[size-1].Calls = append(t.callstack[size-1].Calls, call) @@ -164,22 +207,3 @@ func (t *callTracer) Stop(err error) { t.reason = err atomic.StoreUint32(&t.interrupt, 1) } - -func bytesToHex(s []byte) string { - return "0x" + common.Bytes2Hex(s) -} - -func bigToHex(n *big.Int) string { - if n == nil { - return "" - } - return "0x" + n.Text(16) -} - -func uintToHex(n uint64) string { - return "0x" + strconv.FormatUint(n, 16) -} - -func addrToHex(a common.Address) string { - return strings.ToLower(a.Hex()) -} diff --git a/eth/tracers/native/gen_account_json.go b/eth/tracers/native/gen_account_json.go new file mode 100644 index 0000000000000..25dc77dc74553 --- /dev/null +++ b/eth/tracers/native/gen_account_json.go @@ -0,0 +1,56 @@ +// Code generated by github.com/fjl/gencodec. DO NOT EDIT. + +package native + +import ( + "encoding/json" + "math/big" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" +) + +var _ = (*accountMarshaling)(nil) + +// MarshalJSON marshals as JSON. +func (a account) MarshalJSON() ([]byte, error) { + type account struct { + Balance *hexutil.Big `json:"balance"` + Nonce uint64 `json:"nonce"` + Code hexutil.Bytes `json:"code"` + Storage map[common.Hash]common.Hash `json:"storage"` + } + var enc account + enc.Balance = (*hexutil.Big)(a.Balance) + enc.Nonce = a.Nonce + enc.Code = a.Code + enc.Storage = a.Storage + return json.Marshal(&enc) +} + +// UnmarshalJSON unmarshals from JSON. +func (a *account) UnmarshalJSON(input []byte) error { + type account struct { + Balance *hexutil.Big `json:"balance"` + Nonce *uint64 `json:"nonce"` + Code *hexutil.Bytes `json:"code"` + Storage map[common.Hash]common.Hash `json:"storage"` + } + var dec account + if err := json.Unmarshal(input, &dec); err != nil { + return err + } + if dec.Balance != nil { + a.Balance = (*big.Int)(dec.Balance) + } + if dec.Nonce != nil { + a.Nonce = *dec.Nonce + } + if dec.Code != nil { + a.Code = *dec.Code + } + if dec.Storage != nil { + a.Storage = dec.Storage + } + return nil +} diff --git a/eth/tracers/native/gen_callframe_json.go b/eth/tracers/native/gen_callframe_json.go new file mode 100644 index 0000000000000..bb7658a76ad3e --- /dev/null +++ b/eth/tracers/native/gen_callframe_json.go @@ -0,0 +1,101 @@ +// Code generated by github.com/fjl/gencodec. DO NOT EDIT. + +package native + +import ( + "encoding/json" + "math/big" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/vm" +) + +var _ = (*callFrameMarshaling)(nil) + +// MarshalJSON marshals as JSON. +func (c callFrame) MarshalJSON() ([]byte, error) { + type callFrame0 struct { + Type vm.OpCode `json:"-"` + From common.Address `json:"from"` + Gas hexutil.Uint64 `json:"gas"` + GasUsed hexutil.Uint64 `json:"gasUsed"` + To common.Address `json:"to,omitempty" rlp:"optional"` + Input hexutil.Bytes `json:"input" rlp:"optional"` + Output hexutil.Bytes `json:"output,omitempty" rlp:"optional"` + Error string `json:"error,omitempty" rlp:"optional"` + Revertal string `json:"revertReason,omitempty"` + Calls []callFrame `json:"calls,omitempty" rlp:"optional"` + Value *hexutil.Big `json:"value,omitempty" rlp:"optional"` + TypeString string `json:"type"` + } + var enc callFrame0 + enc.Type = c.Type + enc.From = c.From + enc.Gas = hexutil.Uint64(c.Gas) + enc.GasUsed = hexutil.Uint64(c.GasUsed) + enc.To = c.To + enc.Input = c.Input + enc.Output = c.Output + enc.Error = c.Error + enc.Revertal = c.Revertal + enc.Calls = c.Calls + enc.Value = (*hexutil.Big)(c.Value) + enc.TypeString = c.TypeString() + return json.Marshal(&enc) +} + +// UnmarshalJSON unmarshals from JSON. +func (c *callFrame) UnmarshalJSON(input []byte) error { + type callFrame0 struct { + Type *vm.OpCode `json:"-"` + From *common.Address `json:"from"` + Gas *hexutil.Uint64 `json:"gas"` + GasUsed *hexutil.Uint64 `json:"gasUsed"` + To *common.Address `json:"to,omitempty" rlp:"optional"` + Input *hexutil.Bytes `json:"input" rlp:"optional"` + Output *hexutil.Bytes `json:"output,omitempty" rlp:"optional"` + Error *string `json:"error,omitempty" rlp:"optional"` + Revertal *string `json:"revertReason,omitempty"` + Calls []callFrame `json:"calls,omitempty" rlp:"optional"` + Value *hexutil.Big `json:"value,omitempty" rlp:"optional"` + } + var dec callFrame0 + if err := json.Unmarshal(input, &dec); err != nil { + return err + } + if dec.Type != nil { + c.Type = *dec.Type + } + if dec.From != nil { + c.From = *dec.From + } + if dec.Gas != nil { + c.Gas = uint64(*dec.Gas) + } + if dec.GasUsed != nil { + c.GasUsed = uint64(*dec.GasUsed) + } + if dec.To != nil { + c.To = *dec.To + } + if dec.Input != nil { + c.Input = *dec.Input + } + if dec.Output != nil { + c.Output = *dec.Output + } + if dec.Error != nil { + c.Error = *dec.Error + } + if dec.Revertal != nil { + c.Revertal = *dec.Revertal + } + if dec.Calls != nil { + c.Calls = dec.Calls + } + if dec.Value != nil { + c.Value = (*big.Int)(dec.Value) + } + return nil +} diff --git a/eth/tracers/native/noop.go b/eth/tracers/native/noop.go index 0849fd74e9873..c252b2408fc9b 100644 --- a/eth/tracers/native/noop.go +++ b/eth/tracers/native/noop.go @@ -35,8 +35,8 @@ func init() { type noopTracer struct{} // newNoopTracer returns a new noop tracer. -func newNoopTracer(ctx *tracers.Context) tracers.Tracer { - return &noopTracer{} +func newNoopTracer(ctx *tracers.Context, _ json.RawMessage) (tracers.Tracer, error) { + return &noopTracer{}, nil } // CaptureStart implements the EVMLogger interface to initialize the tracing operation. diff --git a/eth/tracers/native/prestate.go b/eth/tracers/native/prestate.go index 4d289ca622107..a40f84952a4c2 100644 --- a/eth/tracers/native/prestate.go +++ b/eth/tracers/native/prestate.go @@ -29,18 +29,25 @@ import ( "github.com/ethereum/go-ethereum/eth/tracers" ) +//go:generate go run github.com/fjl/gencodec -type account -field-override accountMarshaling -out gen_account_json.go + func init() { register("prestateTracer", newPrestateTracer) } type prestate = map[common.Address]*account type account struct { - Balance string `json:"balance"` + Balance *big.Int `json:"balance"` Nonce uint64 `json:"nonce"` - Code string `json:"code"` + Code []byte `json:"code"` Storage map[common.Hash]common.Hash `json:"storage"` } +type accountMarshaling struct { + Balance *hexutil.Big + Code hexutil.Bytes +} + type prestateTracer struct { env *vm.EVM prestate prestate @@ -51,10 +58,10 @@ type prestateTracer struct { reason error // Textual reason for the interruption } -func newPrestateTracer(ctx *tracers.Context) tracers.Tracer { +func newPrestateTracer(ctx *tracers.Context, _ json.RawMessage) (tracers.Tracer, error) { // First callframe contains tx context info // and is populated on start and end. - return &prestateTracer{prestate: prestate{}} + return &prestateTracer{prestate: prestate{}}, nil } // CaptureStart implements the EVMLogger interface to initialize the tracing operation. @@ -67,17 +74,16 @@ func (t *prestateTracer) CaptureStart(env *vm.EVM, from common.Address, to commo t.lookupAccount(to) // The recipient balance includes the value transferred. - toBal := hexutil.MustDecodeBig(t.prestate[to].Balance) - toBal = new(big.Int).Sub(toBal, value) - t.prestate[to].Balance = hexutil.EncodeBig(toBal) + toBal := new(big.Int).Sub(t.prestate[to].Balance, value) + t.prestate[to].Balance = toBal // The sender balance is after reducing: value and gasLimit. // We need to re-add them to get the pre-tx balance. - fromBal := hexutil.MustDecodeBig(t.prestate[from].Balance) + fromBal := new(big.Int).Set(t.prestate[from].Balance) gasPrice := env.TxContext.GasPrice consumedGas := new(big.Int).Mul(gasPrice, new(big.Int).SetUint64(t.gasLimit)) fromBal.Add(fromBal, new(big.Int).Add(value, consumedGas)) - t.prestate[from].Balance = hexutil.EncodeBig(fromBal) + t.prestate[from].Balance = fromBal t.prestate[from].Nonce-- } @@ -160,9 +166,9 @@ func (t *prestateTracer) lookupAccount(addr common.Address) { return } t.prestate[addr] = &account{ - Balance: bigToHex(t.env.StateDB.GetBalance(addr)), + Balance: t.env.StateDB.GetBalance(addr), Nonce: t.env.StateDB.GetNonce(addr), - Code: bytesToHex(t.env.StateDB.GetCode(addr)), + Code: t.env.StateDB.GetCode(addr), Storage: make(map[common.Hash]common.Hash), } } diff --git a/eth/tracers/native/tracer.go b/eth/tracers/native/tracer.go index 3bab870ea510c..f70d4b2af1ae4 100644 --- a/eth/tracers/native/tracer.go +++ b/eth/tracers/native/tracer.go @@ -14,27 +14,24 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -/* -Package native is a collection of tracers written in go. - -In order to add a native tracer and have it compiled into the binary, a new -file needs to be added to this folder, containing an implementation of the -`eth.tracers.Tracer` interface. - -Aside from implementing the tracer, it also needs to register itself, using the -`register` method -- and this needs to be done in the package initialization. - -Example: - -```golang -func init() { - register("noopTracerNative", newNoopTracer) -} -``` -*/ +// Package native is a collection of tracers written in go. +// +// In order to add a native tracer and have it compiled into the binary, a new +// file needs to be added to this folder, containing an implementation of the +// `eth.tracers.Tracer` interface. +// +// Aside from implementing the tracer, it also needs to register itself, using the +// `register` method -- and this needs to be done in the package initialization. +// +// Example: +// +// func init() { +// register("noopTracerNative", newNoopTracer) +// } package native import ( + "encoding/json" "errors" "github.com/ethereum/go-ethereum/eth/tracers" @@ -46,7 +43,7 @@ func init() { } // ctorFn is the constructor signature of a native tracer. -type ctorFn = func(*tracers.Context) tracers.Tracer +type ctorFn = func(*tracers.Context, json.RawMessage) (tracers.Tracer, error) /* ctors is a map of package-local tracer constructors. @@ -71,12 +68,12 @@ func register(name string, ctor ctorFn) { } // lookup returns a tracer, if one can be matched to the given name. -func lookup(name string, ctx *tracers.Context) (tracers.Tracer, error) { +func lookup(name string, ctx *tracers.Context, cfg json.RawMessage) (tracers.Tracer, error) { if ctors == nil { ctors = make(map[string]ctorFn) } if ctor, ok := ctors[name]; ok { - return ctor(ctx), nil + return ctor(ctx, cfg) } return nil, errors.New("no tracer found") } diff --git a/eth/tracers/tracers.go b/eth/tracers/tracers.go index e7073e7d2edf5..3d2d1256c0914 100644 --- a/eth/tracers/tracers.go +++ b/eth/tracers/tracers.go @@ -42,7 +42,7 @@ type Tracer interface { Stop(err error) } -type lookupFunc func(string, *Context) (Tracer, error) +type lookupFunc func(string, *Context, json.RawMessage) (Tracer, error) var ( lookups []lookupFunc @@ -62,9 +62,9 @@ func RegisterLookup(wildcard bool, lookup lookupFunc) { // New returns a new instance of a tracer, by iterating through the // registered lookups. -func New(code string, ctx *Context) (Tracer, error) { +func New(code string, ctx *Context, cfg json.RawMessage) (Tracer, error) { for _, lookup := range lookups { - if tracer, err := lookup(code, ctx); err == nil { + if tracer, err := lookup(code, ctx, cfg); err == nil { return tracer, nil } } diff --git a/eth/tracers/tracers_test.go b/eth/tracers/tracers_test.go index ce9289dd756b2..12e01abae4034 100644 --- a/eth/tracers/tracers_test.go +++ b/eth/tracers/tracers_test.go @@ -21,7 +21,6 @@ import ( "testing" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" @@ -32,20 +31,6 @@ import ( "github.com/ethereum/go-ethereum/tests" ) -// callTrace is the result of a callTracer run. -type callTrace struct { - Type string `json:"type"` - From common.Address `json:"from"` - To common.Address `json:"to"` - Input hexutil.Bytes `json:"input"` - Output hexutil.Bytes `json:"output"` - Gas *hexutil.Uint64 `json:"gas,omitempty"` - GasUsed *hexutil.Uint64 `json:"gasUsed,omitempty"` - Value *hexutil.Big `json:"value,omitempty"` - Error string `json:"error,omitempty"` - Calls []callTrace `json:"calls,omitempty"` -} - func BenchmarkTransactionTrace(b *testing.B) { key, _ := crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") from := crypto.PubkeyToAddress(key.PublicKey) diff --git a/ethclient/ethclient.go b/ethclient/ethclient.go index 24edd8648ef39..766efcf571408 100644 --- a/ethclient/ethclient.go +++ b/ethclient/ethclient.go @@ -505,6 +505,38 @@ func (ec *Client) SuggestGasTipCap(ctx context.Context) (*big.Int, error) { return (*big.Int)(&hex), nil } +type feeHistoryResultMarshaling struct { + OldestBlock *hexutil.Big `json:"oldestBlock"` + Reward [][]*hexutil.Big `json:"reward,omitempty"` + BaseFee []*hexutil.Big `json:"baseFeePerGas,omitempty"` + GasUsedRatio []float64 `json:"gasUsedRatio"` +} + +// FeeHistory retrieves the fee market history. +func (ec *Client) FeeHistory(ctx context.Context, blockCount uint64, lastBlock *big.Int, rewardPercentiles []float64) (*ethereum.FeeHistory, error) { + var res feeHistoryResultMarshaling + if err := ec.c.CallContext(ctx, &res, "eth_feeHistory", hexutil.Uint(blockCount), toBlockNumArg(lastBlock), rewardPercentiles); err != nil { + return nil, err + } + reward := make([][]*big.Int, len(res.Reward)) + for i, r := range res.Reward { + reward[i] = make([]*big.Int, len(r)) + for j, r := range r { + reward[i][j] = (*big.Int)(r) + } + } + baseFee := make([]*big.Int, len(res.BaseFee)) + for i, b := range res.BaseFee { + baseFee[i] = (*big.Int)(b) + } + return ðereum.FeeHistory{ + OldestBlock: (*big.Int)(res.OldestBlock), + Reward: reward, + BaseFee: baseFee, + GasUsedRatio: res.GasUsedRatio, + }, nil +} + // EstimateGas tries to estimate the gas needed to execute a specific transaction based on // the current pending state of the backend blockchain. There is no guarantee that this is // the true gas limit requirement as other transactions may be added or removed by miners, @@ -538,6 +570,14 @@ func toBlockNumArg(number *big.Int) string { if number.Cmp(pending) == 0 { return "pending" } + finalized := big.NewInt(int64(rpc.FinalizedBlockNumber)) + if number.Cmp(finalized) == 0 { + return "finalized" + } + safe := big.NewInt(int64(rpc.SafeBlockNumber)) + if number.Cmp(safe) == 0 { + return "safe" + } return hexutil.EncodeBig(number) } diff --git a/ethclient/ethclient_test.go b/ethclient/ethclient_test.go index 4a8727b37478e..67b1fde7569cf 100644 --- a/ethclient/ethclient_test.go +++ b/ethclient/ethclient_test.go @@ -30,7 +30,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth" @@ -238,7 +237,6 @@ func newTestBackend(t *testing.T) (*node.Node, []*types.Block) { } func generateTestChain() []*types.Block { - db := rawdb.NewMemoryDatabase() generate := func(i int, g *core.BlockGen) { g.OffsetTime(5) g.SetExtra([]byte("test")) @@ -248,11 +246,8 @@ func generateTestChain() []*types.Block { g.AddTx(testTx2) } } - gblock := genesis.ToBlock(db) - engine := ethash.NewFaker() - blocks, _ := core.GenerateChain(genesis.Config, gblock, engine, db, 2, generate) - blocks = append([]*types.Block{gblock}, blocks...) - return blocks + _, blocks, _ := core.GenerateChainWithGenesis(genesis, ethash.NewFaker(), 2, generate) + return append([]*types.Block{genesis.ToBlock()}, blocks...) } func TestEthClient(t *testing.T) { @@ -508,6 +503,29 @@ func testStatusFunctions(t *testing.T, client *rpc.Client) { if gasTipCap.Cmp(big.NewInt(234375000)) != 0 { t.Fatalf("unexpected gas tip cap: %v", gasTipCap) } + + // FeeHistory + history, err := ec.FeeHistory(context.Background(), 1, big.NewInt(2), []float64{95, 99}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + want := ðereum.FeeHistory{ + OldestBlock: big.NewInt(2), + Reward: [][]*big.Int{ + { + big.NewInt(234375000), + big.NewInt(234375000), + }, + }, + BaseFee: []*big.Int{ + big.NewInt(765625000), + big.NewInt(671627818), + }, + GasUsedRatio: []float64{0.008912678667376286}, + } + if !reflect.DeepEqual(history, want) { + t.Fatalf("FeeHistory result doesn't match expected: (got: %v, want: %v)", history, want) + } } func testCallContractAtHash(t *testing.T, client *rpc.Client) { @@ -558,7 +576,7 @@ func testCallContract(t *testing.T, client *rpc.Client) { if _, err := ec.CallContract(context.Background(), msg, big.NewInt(1)); err != nil { t.Fatalf("unexpected error: %v", err) } - // PendingCallCOntract + // PendingCallContract if _, err := ec.PendingCallContract(context.Background(), msg); err != nil { t.Fatalf("unexpected error: %v", err) } diff --git a/ethclient/gethclient/gethclient.go b/ethclient/gethclient/gethclient.go index 7af2bf45d791e..e182911aa5de8 100644 --- a/ethclient/gethclient/gethclient.go +++ b/ethclient/gethclient/gethclient.go @@ -19,6 +19,7 @@ package gethclient import ( "context" + "encoding/json" "math/big" "runtime" "runtime/debug" @@ -79,7 +80,6 @@ type StorageResult struct { // GetProof returns the account and storage values of the specified account including the Merkle-proof. // The block number can be nil, in which case the value is taken from the latest known block. func (ec *Client) GetProof(ctx context.Context, account common.Address, keys []string, blockNumber *big.Int) (*AccountResult, error) { - type storageResult struct { Key string `json:"key"` Value *hexutil.Big `json:"value"` @@ -119,15 +119,6 @@ func (ec *Client) GetProof(ctx context.Context, account common.Address, keys []s return &result, err } -// OverrideAccount specifies the state of an account to be overridden. -type OverrideAccount struct { - Nonce uint64 `json:"nonce"` - Code []byte `json:"code"` - Balance *big.Int `json:"balance"` - State map[common.Hash]common.Hash `json:"state"` - StateDiff map[common.Hash]common.Hash `json:"stateDiff"` -} - // CallContract executes a message call transaction, which is directly executed in the VM // of the node, but never mined into the blockchain. // @@ -142,7 +133,7 @@ func (ec *Client) CallContract(ctx context.Context, msg ethereum.CallMsg, blockN var hex hexutil.Bytes err := ec.c.CallContext( ctx, &hex, "eth_call", toCallArg(msg), - toBlockNumArg(blockNumber), toOverrideMap(overrides), + toBlockNumArg(blockNumber), overrides, ) return hex, err } @@ -188,6 +179,14 @@ func toBlockNumArg(number *big.Int) string { if number.Cmp(pending) == 0 { return "pending" } + finalized := big.NewInt(int64(rpc.FinalizedBlockNumber)) + if number.Cmp(finalized) == 0 { + return "finalized" + } + safe := big.NewInt(int64(rpc.SafeBlockNumber)) + if number.Cmp(safe) == 0 { + return "safe" + } return hexutil.EncodeBig(number) } @@ -211,26 +210,48 @@ func toCallArg(msg ethereum.CallMsg) interface{} { return arg } -func toOverrideMap(overrides *map[common.Address]OverrideAccount) interface{} { - if overrides == nil { - return nil - } - type overrideAccount struct { - Nonce hexutil.Uint64 `json:"nonce"` - Code hexutil.Bytes `json:"code"` - Balance *hexutil.Big `json:"balance"` - State map[common.Hash]common.Hash `json:"state"` - StateDiff map[common.Hash]common.Hash `json:"stateDiff"` - } - result := make(map[common.Address]overrideAccount) - for addr, override := range *overrides { - result[addr] = overrideAccount{ - Nonce: hexutil.Uint64(override.Nonce), - Code: override.Code, - Balance: (*hexutil.Big)(override.Balance), - State: override.State, - StateDiff: override.StateDiff, - } - } - return &result +// OverrideAccount specifies the state of an account to be overridden. +type OverrideAccount struct { + // Nonce sets nonce of the account. Note: the nonce override will only + // be applied when it is set to a non-zero value. + Nonce uint64 + + // Code sets the contract code. The override will be applied + // when the code is non-nil, i.e. setting empty code is possible + // using an empty slice. + Code []byte + + // Balance sets the account balance. + Balance *big.Int + + // State sets the complete storage. The override will be applied + // when the given map is non-nil. Using an empty map wipes the + // entire contract storage during the call. + State map[common.Hash]common.Hash + + // StateDiff allows overriding individual storage slots. + StateDiff map[common.Hash]common.Hash +} + +func (a OverrideAccount) MarshalJSON() ([]byte, error) { + type acc struct { + Nonce hexutil.Uint64 `json:"nonce,omitempty"` + Code string `json:"code,omitempty"` + Balance *hexutil.Big `json:"balance,omitempty"` + State interface{} `json:"state,omitempty"` + StateDiff map[common.Hash]common.Hash `json:"stateDiff,omitempty"` + } + + output := acc{ + Nonce: hexutil.Uint64(a.Nonce), + Balance: (*hexutil.Big)(a.Balance), + StateDiff: a.StateDiff, + } + if a.Code != nil { + output.Code = hexutil.Encode(a.Code) + } + if a.State != nil { + output.State = a.State + } + return json.Marshal(output) } diff --git a/ethclient/gethclient/gethclient_test.go b/ethclient/gethclient/gethclient_test.go index 758acc085b377..0e2f7e57b69a9 100644 --- a/ethclient/gethclient/gethclient_test.go +++ b/ethclient/gethclient/gethclient_test.go @@ -19,6 +19,7 @@ package gethclient import ( "bytes" "context" + "encoding/json" "math/big" "testing" @@ -26,11 +27,11 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/eth/ethconfig" + "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/params" @@ -60,6 +61,12 @@ func newTestBackend(t *testing.T) (*node.Node, []*types.Block) { if err != nil { t.Fatalf("can't create new ethereum service: %v", err) } + filterSystem := filters.NewFilterSystem(ethservice.APIBackend, filters.Config{}) + n.RegisterAPIs([]rpc.API{{ + Namespace: "eth", + Service: filters.NewFilterAPI(filterSystem, false), + }}) + // Import the test chain. if err := n.Start(); err != nil { t.Fatalf("can't start test node: %v", err) @@ -71,10 +78,8 @@ func newTestBackend(t *testing.T) (*node.Node, []*types.Block) { } func generateTestChain() (*core.Genesis, []*types.Block) { - db := rawdb.NewMemoryDatabase() - config := params.AllEthashProtocolChanges genesis := &core.Genesis{ - Config: config, + Config: params.AllEthashProtocolChanges, Alloc: core.GenesisAlloc{testAddr: {Balance: testBalance, Storage: map[common.Hash]common.Hash{testSlot: testValue}}}, ExtraData: []byte("test genesis"), Timestamp: 9000, @@ -83,10 +88,8 @@ func generateTestChain() (*core.Genesis, []*types.Block) { g.OffsetTime(5) g.SetExtra([]byte("test")) } - gblock := genesis.ToBlock(db) - engine := ethash.NewFaker() - blocks, _ := core.GenerateChain(config, gblock, engine, db, 1, generate) - blocks = append([]*types.Block{gblock}, blocks...) + _, blocks, _ := core.GenerateChainWithGenesis(genesis, ethash.NewFaker(), 1, generate) + blocks = append([]*types.Block{genesis.ToBlock()}, blocks...) return genesis, blocks } @@ -222,7 +225,6 @@ func testGetProof(t *testing.T, client *rpc.Client) { if proof.Key != testSlot.String() { t.Fatalf("invalid storage proof key, want: %v, got: %v", testSlot.String(), proof.Key) } - } func testGCStats(t *testing.T, client *rpc.Client) { @@ -321,3 +323,53 @@ func testCallContract(t *testing.T, client *rpc.Client) { t.Fatalf("unexpected error: %v", err) } } + +func TestOverrideAccountMarshal(t *testing.T) { + om := map[common.Address]OverrideAccount{ + common.Address{0x11}: OverrideAccount{ + // Zero-valued nonce is not overriddden, but simply dropped by the encoder. + Nonce: 0, + }, + common.Address{0xaa}: OverrideAccount{ + Nonce: 5, + }, + common.Address{0xbb}: OverrideAccount{ + Code: []byte{1}, + }, + common.Address{0xcc}: OverrideAccount{ + // 'code', 'balance', 'state' should be set when input is + // a non-nil but empty value. + Code: []byte{}, + Balance: big.NewInt(0), + State: map[common.Hash]common.Hash{}, + // For 'stateDiff' the behavior is different, empty map + // is ignored because it makes no difference. + StateDiff: map[common.Hash]common.Hash{}, + }, + } + + marshalled, err := json.MarshalIndent(&om, "", " ") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + expected := `{ + "0x1100000000000000000000000000000000000000": {}, + "0xaa00000000000000000000000000000000000000": { + "nonce": "0x5" + }, + "0xbb00000000000000000000000000000000000000": { + "code": "0x01" + }, + "0xcc00000000000000000000000000000000000000": { + "code": "0x", + "balance": "0x0", + "state": {} + } +}` + + if string(marshalled) != expected { + t.Error("wrong output:", string(marshalled)) + t.Error("want:", expected) + } +} diff --git a/ethdb/database.go b/ethdb/database.go index e8faa2d868cc8..361218f247421 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -142,7 +142,9 @@ type AncientWriteOp interface { // AncientStater wraps the Stat method of a backing data store. type AncientStater interface { - // AncientDatadir returns the root directory path of the ancient store. + // AncientDatadir returns the path of root ancient directory. Empty string + // will be returned if ancient store is not enabled at all. The returned + // path can be used to construct the path of other freezers. AncientDatadir() (string, error) } @@ -172,7 +174,6 @@ type Stater interface { type AncientStore interface { AncientReader AncientWriter - AncientStater io.Closer } diff --git a/ethdb/leveldb/leveldb.go b/ethdb/leveldb/leveldb.go index 15bd4e6eb3b59..0467531721c27 100644 --- a/ethdb/leveldb/leveldb.go +++ b/ethdb/leveldb/leveldb.go @@ -266,13 +266,14 @@ func (db *Database) Path() string { // the metrics subsystem. // // This is how a LevelDB stats table looks like (currently): -// Compactions -// Level | Tables | Size(MB) | Time(sec) | Read(MB) | Write(MB) -// -------+------------+---------------+---------------+---------------+--------------- -// 0 | 0 | 0.00000 | 1.27969 | 0.00000 | 12.31098 -// 1 | 85 | 109.27913 | 28.09293 | 213.92493 | 214.26294 -// 2 | 523 | 1000.37159 | 7.26059 | 66.86342 | 66.77884 -// 3 | 570 | 1113.18458 | 0.00000 | 0.00000 | 0.00000 +// +// Compactions +// Level | Tables | Size(MB) | Time(sec) | Read(MB) | Write(MB) +// -------+------------+---------------+---------------+---------------+--------------- +// 0 | 0 | 0.00000 | 1.27969 | 0.00000 | 12.31098 +// 1 | 85 | 109.27913 | 28.09293 | 213.92493 | 214.26294 +// 2 | 523 | 1000.37159 | 7.26059 | 66.86342 | 66.77884 +// 3 | 570 | 1113.18458 | 0.00000 | 0.00000 | 0.00000 // // This is how the write delay look like (currently): // DelayN:5 Delay:406.604657ms Paused: false diff --git a/ethdb/memorydb/memorydb.go b/ethdb/memorydb/memorydb.go index e94570cb3f0ec..7e4fd7e5e7f0e 100644 --- a/ethdb/memorydb/memorydb.go +++ b/ethdb/memorydb/memorydb.go @@ -66,7 +66,7 @@ func NewWithCap(size int) *Database { } // Close deallocates the internal map and ensures any consecutive data access op -// failes with an error. +// fails with an error. func (db *Database) Close() error { db.lock.Lock() defer db.lock.Unlock() diff --git a/ethdb/remotedb/remotedb.go b/ethdb/remotedb/remotedb.go index 59a570bb5e96a..9ce657d78026e 100644 --- a/ethdb/remotedb/remotedb.go +++ b/ethdb/remotedb/remotedb.go @@ -22,9 +22,6 @@ package remotedb import ( - "errors" - "strings" - "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/rpc" @@ -150,24 +147,8 @@ func (db *Database) Close() error { return nil } -func dialRPC(endpoint string) (*rpc.Client, error) { - if endpoint == "" { - return nil, errors.New("endpoint must be specified") - } - if strings.HasPrefix(endpoint, "rpc:") || strings.HasPrefix(endpoint, "ipc:") { - // Backwards compatibility with geth < 1.5 which required - // these prefixes. - endpoint = endpoint[4:] - } - return rpc.Dial(endpoint) -} - -func New(endpoint string) (ethdb.Database, error) { - client, err := dialRPC(endpoint) - if err != nil { - return nil, err - } +func New(client *rpc.Client) ethdb.Database { return &Database{ remote: client, - }, nil + } } diff --git a/ethstats/ethstats.go b/ethstats/ethstats.go index 5d60efab2ec1d..f6ad360519c64 100644 --- a/ethstats/ethstats.go +++ b/ethstats/ethstats.go @@ -102,13 +102,17 @@ type Service struct { // websocket. // // From Gorilla websocket docs: -// Connections support one concurrent reader and one concurrent writer. -// Applications are responsible for ensuring that no more than one goroutine calls the write methods -// - NextWriter, SetWriteDeadline, WriteMessage, WriteJSON, EnableWriteCompression, SetCompressionLevel -// concurrently and that no more than one goroutine calls the read methods -// - NextReader, SetReadDeadline, ReadMessage, ReadJSON, SetPongHandler, SetPingHandler -// concurrently. -// The Close and WriteControl methods can be called concurrently with all other methods. +// +// Connections support one concurrent reader and one concurrent writer. Applications are +// responsible for ensuring that +// - no more than one goroutine calls the write methods +// NextWriter, SetWriteDeadline, WriteMessage, WriteJSON, EnableWriteCompression, +// SetCompressionLevel concurrently; and +// - that no more than one goroutine calls the +// read methods NextReader, SetReadDeadline, ReadMessage, ReadJSON, SetPongHandler, +// SetPingHandler concurrently. +// +// The Close and WriteControl methods can be called concurrently with all other methods. type connWrapper struct { conn *websocket.Conn diff --git a/ethstats/ethstats_test.go b/ethstats/ethstats_test.go index 0692ecdae9bec..60322f765439e 100644 --- a/ethstats/ethstats_test.go +++ b/ethstats/ethstats_test.go @@ -79,5 +79,4 @@ func TestParseEthstatsURL(t *testing.T) { t.Errorf("case=%d mismatch host value, got: %v ,want: %v", i, host, c.host) } } - } diff --git a/go.mod b/go.mod index fc944282f2d74..72613343948eb 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff github.com/go-stack/stack v1.8.0 github.com/golang-jwt/jwt/v4 v4.3.0 - github.com/golang/protobuf v1.4.3 + github.com/golang/protobuf v1.5.2 github.com/golang/snappy v0.0.4 github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa github.com/google/uuid v1.2.0 @@ -51,19 +51,20 @@ require ( github.com/rs/cors v1.7.0 github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 - github.com/stretchr/testify v1.7.0 + github.com/stretchr/testify v1.7.2 github.com/supranational/blst v0.3.8-0.20220526154634-513d2456b344 github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef + github.com/urfave/cli/v2 v2.10.2 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c - golang.org/x/sys v0.0.0-20211019181941-9d821ace8654 + golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a + golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 golang.org/x/text v0.3.7 golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023 gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce - gopkg.in/urfave/cli.v1 v1.20.0 ) require ( @@ -76,13 +77,18 @@ require ( github.com/aws/aws-sdk-go-v2/service/sts v1.1.1 // indirect github.com/aws/smithy-go v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.1.1 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect + github.com/crate-crypto/go-ipa v0.0.0-20220523130400-f11357ae11c7 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect github.com/deepmap/oapi-codegen v1.8.2 // indirect github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91 // indirect github.com/garslo/gogen v0.0.0-20170306192744-1d203ffc1f61 // indirect + github.com/gballet/go-verkle v0.0.0-20220902153445-097bd83b7732 // indirect + github.com/go-logfmt/logfmt v0.4.0 // indirect github.com/go-ole/go-ole v1.2.1 // indirect github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097 // indirect + github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/mattn/go-runewidth v0.0.9 // indirect github.com/mitchellh/mapstructure v1.4.1 // indirect @@ -91,13 +97,14 @@ require ( github.com/opentracing/opentracing-go v1.1.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/tklauser/go-sysconf v0.3.5 // indirect github.com/tklauser/numcpus v0.2.2 // indirect + github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57 // indirect - golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f // indirect - golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 // indirect - golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect - google.golang.org/protobuf v1.23.0 // indirect + golang.org/x/net v0.0.0-20220607020251-c690dde0001d // indirect + golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df // indirect + google.golang.org/protobuf v1.26.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 8d28443f6c615..72d9b25e2021b 100644 --- a/go.sum +++ b/go.sum @@ -25,6 +25,7 @@ github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.3/go.mod h1:KLF4gFr6DcKFZwSu github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0 h1:Px2UA+2RvSSvv+RvJNuUB6n7rs5Wsel4dXLe90Um2n4= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0/go.mod h1:tPaiy8S5bQ+S5sOiDlINkp7+Ef339+Nz5L5XO+cnOHo= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v1.1.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= @@ -83,6 +84,10 @@ github.com/consensys/bavard v0.1.8-0.20210406032232-f3452dc9b572/go.mod h1:Bpd0/ github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f h1:C43yEtQ6NIf4ftFXD/V55gnGFgPbMQobd//YlnLjUJ8= github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f/go.mod h1:815PAHg3wvysy0SyIqanF8gZ0Y1wjk/hrDHD/iT88+Q= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/crate-crypto/go-ipa v0.0.0-20220523130400-f11357ae11c7 h1:6IrxszG5G+O7zhtkWxq6+unVvnrm1fqV2Pe+T95DUzw= +github.com/crate-crypto/go-ipa v0.0.0-20220523130400-f11357ae11c7/go.mod h1:gFnFS95y8HstDP6P9pPwzrxOOC5TRDkwbM+ao15ChAI= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= @@ -124,12 +129,15 @@ github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 h1:FtmdgXiUlNeRsoNMFlK github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= +github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/garslo/gogen v0.0.0-20170306192744-1d203ffc1f61 h1:IZqZOB2fydHte3kUgxrzK5E1fW7RQGeDwE8F/ZZnUYc= github.com/garslo/gogen v0.0.0-20170306192744-1d203ffc1f61/go.mod h1:Q0X6pkwTILDlzrGEckF6HKjXe48EgsY/l7K7vhY4MW8= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff h1:tY80oXqGNY4FhTFhk+o9oFHGINQ/+vhlm8HFzi6znCI= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= +github.com/gballet/go-verkle v0.0.0-20220902153445-097bd83b7732 h1:AB7YjNrzlVHsYz06zCULVV2zYCEft82P86dSmtwxKL0= +github.com/gballet/go-verkle v0.0.0-20220902153445-097bd83b7732/go.mod h1:o/XfIXWi4/GqbQirfRm5uTbXMG5NpqxkxblnbZ+QM9I= github.com/getkin/kin-openapi v0.53.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= github.com/getkin/kin-openapi v0.61.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -152,6 +160,7 @@ github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5Nq github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= @@ -174,8 +183,9 @@ github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrU github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= @@ -189,14 +199,16 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa h1:Q75Upo5UN4JbPFURXZ8nLKYUvF85dyFRop/vQ0Rv+64= github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -223,6 +235,7 @@ github.com/huin/goupnp v1.0.3 h1:N8No57ls+MnjlB+JPiCVSOyy/ot7MJTqlo7rn+NYSqQ= github.com/huin/goupnp v1.0.3/go.mod h1:ZxNlw5WqJj6wSsRK5+YfflQGXYfccj5VgQsMNixHM7Y= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/flux v0.65.1/go.mod h1:J754/zds0vvpfwuq7Gc2wRdVwEodfpCFM7mYlOw2LqY= github.com/influxdata/influxdb v1.8.3 h1:WEypI1BQFTT4teLM+1qkEcvUi0dAvopAI/ir0vAiBg8= @@ -311,19 +324,29 @@ github.com/naoina/go-stringutil v0.1.0 h1:rCUeRUHjBjGTSHl0VC00jUPLz8/F9dDzYI70Hz github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0= github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416 h1:shk/vn9oCoOTmwcouEdwIeOtOGA/ELRUw/GwvxwfT+0= github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= -github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.3 h1:OoxbjfXVZyod1fmWYhI7SEyaD8B00ynP3T+D5GiyHOY= github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.1 h1:K0jcRCwNQM3vFGh1ppMtDh/+7ApJrjldlX8fA0jDTLQ= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw= +github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.0.3-0.20180606204148-bd9c31933947/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= @@ -360,6 +383,8 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/segmentio/kafka-go v0.1.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= github.com/segmentio/kafka-go v0.2.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= @@ -382,12 +407,19 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.2 h1:4jaiDzPyXQvSd7D0EjG45355tLlV3VOECpq10pLC+8s= +github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/supranational/blst v0.3.8-0.20220526154634-513d2456b344 h1:m+8fKfQwCAy1QjzINvKe/pYtLjo2dl59x2w9YSEJxuY= github.com/supranational/blst v0.3.8-0.20220526154634-513d2456b344/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= +github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= +github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= +github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a h1:1ur3QoCqvE5fl+nylMaIr9PVV1w343YRDtsy+Rwu7XI= +github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a/go.mod h1:RRCYJbIwD5jmqPI9XoAFR0OcDxqUctll6zUj/+B4S48= +github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d h1:vfofYNRScrDdvS342BElfbETmL1Aiz3i2t0zfRj16Hs= +github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d/go.mod h1:RRCYJbIwD5jmqPI9XoAFR0OcDxqUctll6zUj/+B4S48= github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tklauser/go-sysconf v0.3.5 h1:uu3Xl4nkLzQfXNsWn15rPc/HQCJKObbt1dKJeWp3vU4= github.com/tklauser/go-sysconf v0.3.5/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI= @@ -396,11 +428,15 @@ github.com/tklauser/numcpus v0.2.2/go.mod h1:x3qojaO3uyYt0i56EW/VUYs7uBvdl2fkfZF github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef h1:wHSqTBrZW24CsNJDfeh9Ex6Pm0Rcpc7qrgKBiL44vF4= github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= +github.com/urfave/cli/v2 v2.10.2 h1:x3p8awjp/2arX+Nl/G2040AZpOCHS/eMJJ1/a+mye4Y= +github.com/urfave/cli/v2 v2.10.2/go.mod h1:f8iq5LtQ/bLxafbdBSLPPNsgaW0l/2fYYEHhAyPlwvo= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= +github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= +github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= @@ -477,10 +513,13 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210220033124-5f55cee0dc0d/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f h1:OfiFi4JbukWwe3lzw+xunroH1mnC1e2Gy5cxNJApiSY= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220607020251-c690dde0001d h1:4SFsTMi4UahlKoloni7L4eYzhFRifURQLw+yv0QDCx8= +golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -525,6 +564,7 @@ golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200826173525-f9321e4c35a6/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -532,11 +572,16 @@ golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210420205809-ac73e9fd8988/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211019181941-9d821ace8654 h1:id054HUawV2/6IGm2IV8KZQjqtwAOo2CYlOToYqa0d0= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211020174200-9d6173849985/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -578,14 +623,16 @@ golang.org/x/tools v0.0.0-20191126055441-b0650ceb63d9/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200108203644-89082a384178/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023 h1:0c3L82FDQ5rt1bjTBlchS8t6RQ6299/+5bWMnRLh+uI= golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df h1:5Pf6pFKu98ODmgnpvkJ3kFUOQGGLIzLIkbzUHp47618= +golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.6.0/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU= @@ -628,8 +675,10 @@ google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -641,8 +690,6 @@ gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7 gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/urfave/cli.v1 v1.20.0 h1:NdAVW6RYxDif9DhDHaAortIu956m2c0v+09AZBPTbE0= -gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -652,8 +699,9 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/graphql/graphql.go b/graphql/graphql.go index 0654fd1af3885..356ff669fb16d 100644 --- a/graphql/graphql.go +++ b/graphql/graphql.go @@ -22,6 +22,7 @@ import ( "errors" "fmt" "math/big" + "sort" "strconv" "github.com/ethereum/go-ethereum" @@ -76,14 +77,14 @@ func (b *Long) UnmarshalGraphQL(input interface{}) error { // Account represents an Ethereum account at a particular block. type Account struct { - backend ethapi.Backend + r *Resolver address common.Address blockNrOrHash rpc.BlockNumberOrHash } // getState fetches the StateDB object for an account. func (a *Account) getState(ctx context.Context) (*state.StateDB, error) { - state, _, err := a.backend.StateAndHeaderByNumberOrHash(ctx, a.blockNrOrHash) + state, _, err := a.r.backend.StateAndHeaderByNumberOrHash(ctx, a.blockNrOrHash) return state, err } @@ -106,7 +107,7 @@ func (a *Account) Balance(ctx context.Context) (hexutil.Big, error) { func (a *Account) TransactionCount(ctx context.Context) (hexutil.Uint64, error) { // Ask transaction pool for the nonce which includes pending transactions if blockNr, ok := a.blockNrOrHash.Number(); ok && blockNr == rpc.PendingBlockNumber { - nonce, err := a.backend.GetPoolNonce(ctx, a.address) + nonce, err := a.r.backend.GetPoolNonce(ctx, a.address) if err != nil { return 0, err } @@ -137,7 +138,7 @@ func (a *Account) Storage(ctx context.Context, args struct{ Slot common.Hash }) // Log represents an individual log message. All arguments are mandatory. type Log struct { - backend ethapi.Backend + r *Resolver transaction *Transaction log *types.Log } @@ -148,7 +149,7 @@ func (l *Log) Transaction(ctx context.Context) *Transaction { func (l *Log) Account(ctx context.Context, args BlockNumberArgs) *Account { return &Account{ - backend: l.backend, + r: l.r, address: l.log.Address, blockNrOrHash: args.NumberOrLatest(), } @@ -183,30 +184,30 @@ func (at *AccessTuple) StorageKeys(ctx context.Context) []common.Hash { // Transaction represents an Ethereum transaction. // backend and hash are mandatory; all others will be fetched when required. type Transaction struct { - backend ethapi.Backend - hash common.Hash - tx *types.Transaction - block *Block - index uint64 + r *Resolver + hash common.Hash + tx *types.Transaction + block *Block + index uint64 } // resolve returns the internal transaction object, fetching it if needed. func (t *Transaction) resolve(ctx context.Context) (*types.Transaction, error) { if t.tx == nil { // Try to return an already finalized transaction - tx, blockHash, _, index, err := t.backend.GetTransaction(ctx, t.hash) + tx, blockHash, _, index, err := t.r.backend.GetTransaction(ctx, t.hash) if err == nil && tx != nil { t.tx = tx blockNrOrHash := rpc.BlockNumberOrHashWithHash(blockHash, false) t.block = &Block{ - backend: t.backend, + r: t.r, numberOrHash: &blockNrOrHash, } t.index = index return t.tx, nil } // No finalized transaction, try to retrieve it from the pool - t.tx = t.backend.GetPoolTransaction(t.hash) + t.tx = t.r.backend.GetPoolTransaction(t.hash) } return t.tx, nil } @@ -354,7 +355,7 @@ func (t *Transaction) To(ctx context.Context, args BlockNumberArgs) (*Account, e return nil, nil } return &Account{ - backend: t.backend, + r: t.r, address: *to, blockNrOrHash: args.NumberOrLatest(), }, nil @@ -365,10 +366,10 @@ func (t *Transaction) From(ctx context.Context, args BlockNumberArgs) (*Account, if err != nil || tx == nil { return nil, err } - signer := types.LatestSigner(t.backend.ChainConfig()) + signer := types.LatestSigner(t.r.backend.ChainConfig()) from, _ := types.Sender(signer, tx) return &Account{ - backend: t.backend, + r: t.r, address: from, blockNrOrHash: args.NumberOrLatest(), }, nil @@ -443,24 +444,51 @@ func (t *Transaction) CreatedContract(ctx context.Context, args BlockNumberArgs) return nil, err } return &Account{ - backend: t.backend, + r: t.r, address: receipt.ContractAddress, blockNrOrHash: args.NumberOrLatest(), }, nil } func (t *Transaction) Logs(ctx context.Context) (*[]*Log, error) { - receipt, err := t.getReceipt(ctx) - if err != nil || receipt == nil { + if _, err := t.resolve(ctx); err != nil { + return nil, err + } + if t.block == nil { + return nil, nil + } + if _, ok := t.block.numberOrHash.Hash(); !ok { + header, err := t.r.backend.HeaderByNumberOrHash(ctx, *t.block.numberOrHash) + if err != nil { + return nil, err + } + hash := header.Hash() + t.block.numberOrHash.BlockHash = &hash + } + return t.getLogs(ctx) +} + +// getLogs returns log objects for the given tx. +// Assumes block hash is resolved. +func (t *Transaction) getLogs(ctx context.Context) (*[]*Log, error) { + var ( + hash, _ = t.block.numberOrHash.Hash() + filter = t.r.filterSystem.NewBlockFilter(hash, nil, nil) + logs, err = filter.Logs(ctx) + ) + if err != nil { return nil, err } - ret := make([]*Log, 0, len(receipt.Logs)) - for _, log := range receipt.Logs { + var ret []*Log + // Select tx logs from all block logs + ix := sort.Search(len(logs), func(i int) bool { return uint64(logs[i].TxIndex) >= t.index }) + for ix < len(logs) && uint64(logs[ix].TxIndex) == t.index { ret = append(ret, &Log{ - backend: t.backend, + r: t.r, transaction: t, - log: log, + log: logs[ix], }) + ix++ } return &ret, nil } @@ -539,7 +567,7 @@ type BlockType int // backend, and numberOrHash are mandatory. All other fields are lazily fetched // when required. type Block struct { - backend ethapi.Backend + r *Resolver numberOrHash *rpc.BlockNumberOrHash hash common.Hash header *types.Header @@ -558,7 +586,7 @@ func (b *Block) resolve(ctx context.Context) (*types.Block, error) { b.numberOrHash = &latest } var err error - b.block, err = b.backend.BlockByNumberOrHash(ctx, *b.numberOrHash) + b.block, err = b.r.backend.BlockByNumberOrHash(ctx, *b.numberOrHash) if b.block != nil && b.header == nil { b.header = b.block.Header() if hash, ok := b.numberOrHash.Hash(); ok { @@ -578,9 +606,9 @@ func (b *Block) resolveHeader(ctx context.Context) (*types.Header, error) { var err error if b.header == nil { if b.hash != (common.Hash{}) { - b.header, err = b.backend.HeaderByHash(ctx, b.hash) + b.header, err = b.r.backend.HeaderByHash(ctx, b.hash) } else { - b.header, err = b.backend.HeaderByNumberOrHash(ctx, *b.numberOrHash) + b.header, err = b.r.backend.HeaderByNumberOrHash(ctx, *b.numberOrHash) } } return b.header, err @@ -598,7 +626,7 @@ func (b *Block) resolveReceipts(ctx context.Context) ([]*types.Receipt, error) { } hash = header.Hash() } - receipts, err := b.backend.GetReceipts(ctx, hash) + receipts, err := b.r.backend.GetReceipts(ctx, hash) if err != nil { return nil, err } @@ -659,7 +687,7 @@ func (b *Block) NextBaseFeePerGas(ctx context.Context) (*hexutil.Big, error) { if err != nil { return nil, err } - chaincfg := b.backend.ChainConfig() + chaincfg := b.r.backend.ChainConfig() if header.BaseFee == nil { // Make sure next block doesn't enable EIP-1559 if !chaincfg.IsLondon(new(big.Int).Add(header.Number, common.Big1)) { @@ -679,7 +707,7 @@ func (b *Block) Parent(ctx context.Context) (*Block, error) { } num := rpc.BlockNumberOrHashWithNumber(rpc.BlockNumber(b.header.Number.Uint64() - 1)) return &Block{ - backend: b.backend, + r: b.r, numberOrHash: &num, hash: b.header.ParentHash, }, nil @@ -767,7 +795,7 @@ func (b *Block) Ommers(ctx context.Context) (*[]*Block, error) { for _, uncle := range block.Uncles() { blockNumberOrHash := rpc.BlockNumberOrHashWithHash(uncle.Hash(), false) ret = append(ret, &Block{ - backend: b.backend, + r: b.r, numberOrHash: &blockNumberOrHash, header: uncle, }) @@ -800,7 +828,7 @@ func (b *Block) TotalDifficulty(ctx context.Context) (hexutil.Big, error) { } h = header.Hash() } - td := b.backend.GetTd(ctx, h) + td := b.r.backend.GetTd(ctx, h) if td == nil { return hexutil.Big{}, fmt.Errorf("total difficulty not found %x", b.hash) } @@ -853,7 +881,7 @@ func (b *Block) Miner(ctx context.Context, args BlockNumberArgs) (*Account, erro return nil, err } return &Account{ - backend: b.backend, + r: b.r, address: header.Coinbase, blockNrOrHash: args.NumberOrLatest(), }, nil @@ -876,11 +904,11 @@ func (b *Block) Transactions(ctx context.Context) (*[]*Transaction, error) { ret := make([]*Transaction, 0, len(block.Transactions())) for i, tx := range block.Transactions() { ret = append(ret, &Transaction{ - backend: b.backend, - hash: tx.Hash(), - tx: tx, - block: b, - index: uint64(i), + r: b.r, + hash: tx.Hash(), + tx: tx, + block: b, + index: uint64(i), }) } return &ret, nil @@ -897,11 +925,11 @@ func (b *Block) TransactionAt(ctx context.Context, args struct{ Index int32 }) ( } tx := txs[args.Index] return &Transaction{ - backend: b.backend, - hash: tx.Hash(), - tx: tx, - block: b, - index: uint64(args.Index), + r: b.r, + hash: tx.Hash(), + tx: tx, + block: b, + index: uint64(args.Index), }, nil } @@ -917,7 +945,7 @@ func (b *Block) OmmerAt(ctx context.Context, args struct{ Index int32 }) (*Block uncle := uncles[args.Index] blockNumberOrHash := rpc.BlockNumberOrHashWithHash(uncle.Hash(), false) return &Block{ - backend: b.backend, + r: b.r, numberOrHash: &blockNumberOrHash, header: uncle, }, nil @@ -944,7 +972,7 @@ type BlockFilterCriteria struct { // runFilter accepts a filter and executes it, returning all its results as // `Log` objects. -func runFilter(ctx context.Context, be ethapi.Backend, filter *filters.Filter) ([]*Log, error) { +func runFilter(ctx context.Context, r *Resolver, filter *filters.Filter) ([]*Log, error) { logs, err := filter.Logs(ctx) if err != nil || logs == nil { return nil, err @@ -952,8 +980,8 @@ func runFilter(ctx context.Context, be ethapi.Backend, filter *filters.Filter) ( ret := make([]*Log, 0, len(logs)) for _, log := range logs { ret = append(ret, &Log{ - backend: be, - transaction: &Transaction{backend: be, hash: log.TxHash}, + r: r, + transaction: &Transaction{r: r, hash: log.TxHash}, log: log, }) } @@ -978,10 +1006,10 @@ func (b *Block) Logs(ctx context.Context, args struct{ Filter BlockFilterCriteri hash = header.Hash() } // Construct the range filter - filter := filters.NewBlockFilter(b.backend, hash, addresses, topics) + filter := b.r.filterSystem.NewBlockFilter(hash, addresses, topics) // Run the filter and return all the logs - return runFilter(ctx, b.backend, filter) + return runFilter(ctx, b.r, filter) } func (b *Block) Account(ctx context.Context, args struct { @@ -994,7 +1022,7 @@ func (b *Block) Account(ctx context.Context, args struct { } } return &Account{ - backend: b.backend, + r: b.r, address: args.Address, blockNrOrHash: *b.numberOrHash, }, nil @@ -1041,7 +1069,7 @@ func (b *Block) Call(ctx context.Context, args struct { return nil, err } } - result, err := ethapi.DoCall(ctx, b.backend, args.Data, *b.numberOrHash, nil, b.backend.RPCEVMTimeout(), b.backend.RPCGasCap()) + result, err := ethapi.DoCall(ctx, b.r.backend, args.Data, *b.numberOrHash, nil, b.r.backend.RPCEVMTimeout(), b.r.backend.RPCGasCap()) if err != nil { return nil, err } @@ -1066,31 +1094,31 @@ func (b *Block) EstimateGas(ctx context.Context, args struct { return 0, err } } - gas, err := ethapi.DoEstimateGas(ctx, b.backend, args.Data, *b.numberOrHash, b.backend.RPCGasCap()) + gas, err := ethapi.DoEstimateGas(ctx, b.r.backend, args.Data, *b.numberOrHash, b.r.backend.RPCGasCap()) return Long(gas), err } type Pending struct { - backend ethapi.Backend + r *Resolver } func (p *Pending) TransactionCount(ctx context.Context) (int32, error) { - txs, err := p.backend.GetPoolTransactions() + txs, err := p.r.backend.GetPoolTransactions() return int32(len(txs)), err } func (p *Pending) Transactions(ctx context.Context) (*[]*Transaction, error) { - txs, err := p.backend.GetPoolTransactions() + txs, err := p.r.backend.GetPoolTransactions() if err != nil { return nil, err } ret := make([]*Transaction, 0, len(txs)) for i, tx := range txs { ret = append(ret, &Transaction{ - backend: p.backend, - hash: tx.Hash(), - tx: tx, - index: uint64(i), + r: p.r, + hash: tx.Hash(), + tx: tx, + index: uint64(i), }) } return &ret, nil @@ -1101,7 +1129,7 @@ func (p *Pending) Account(ctx context.Context, args struct { }) *Account { pendingBlockNr := rpc.BlockNumberOrHashWithNumber(rpc.PendingBlockNumber) return &Account{ - backend: p.backend, + r: p.r, address: args.Address, blockNrOrHash: pendingBlockNr, } @@ -1111,7 +1139,7 @@ func (p *Pending) Call(ctx context.Context, args struct { Data ethapi.TransactionArgs }) (*CallResult, error) { pendingBlockNr := rpc.BlockNumberOrHashWithNumber(rpc.PendingBlockNumber) - result, err := ethapi.DoCall(ctx, p.backend, args.Data, pendingBlockNr, nil, p.backend.RPCEVMTimeout(), p.backend.RPCGasCap()) + result, err := ethapi.DoCall(ctx, p.r.backend, args.Data, pendingBlockNr, nil, p.r.backend.RPCEVMTimeout(), p.r.backend.RPCGasCap()) if err != nil { return nil, err } @@ -1131,13 +1159,14 @@ func (p *Pending) EstimateGas(ctx context.Context, args struct { Data ethapi.TransactionArgs }) (Long, error) { pendingBlockNr := rpc.BlockNumberOrHashWithNumber(rpc.PendingBlockNumber) - gas, err := ethapi.DoEstimateGas(ctx, p.backend, args.Data, pendingBlockNr, p.backend.RPCGasCap()) + gas, err := ethapi.DoEstimateGas(ctx, p.r.backend, args.Data, pendingBlockNr, p.r.backend.RPCGasCap()) return Long(gas), err } // Resolver is the top-level object in the GraphQL hierarchy. type Resolver struct { - backend ethapi.Backend + backend ethapi.Backend + filterSystem *filters.FilterSystem } func (r *Resolver) Block(ctx context.Context, args struct { @@ -1152,19 +1181,19 @@ func (r *Resolver) Block(ctx context.Context, args struct { number := rpc.BlockNumber(*args.Number) numberOrHash := rpc.BlockNumberOrHashWithNumber(number) block = &Block{ - backend: r.backend, + r: r, numberOrHash: &numberOrHash, } } else if args.Hash != nil { numberOrHash := rpc.BlockNumberOrHashWithHash(*args.Hash, false) block = &Block{ - backend: r.backend, + r: r, numberOrHash: &numberOrHash, } } else { numberOrHash := rpc.BlockNumberOrHashWithNumber(rpc.LatestBlockNumber) block = &Block{ - backend: r.backend, + r: r, numberOrHash: &numberOrHash, } } @@ -1199,7 +1228,7 @@ func (r *Resolver) Blocks(ctx context.Context, args struct { for i := from; i <= to; i++ { numberOrHash := rpc.BlockNumberOrHashWithNumber(i) block := &Block{ - backend: r.backend, + r: r, numberOrHash: &numberOrHash, } // Resolve the header to check for existence. @@ -1218,13 +1247,13 @@ func (r *Resolver) Blocks(ctx context.Context, args struct { } func (r *Resolver) Pending(ctx context.Context) *Pending { - return &Pending{r.backend} + return &Pending{r} } func (r *Resolver) Transaction(ctx context.Context, args struct{ Hash common.Hash }) (*Transaction, error) { tx := &Transaction{ - backend: r.backend, - hash: args.Hash, + r: r, + hash: args.Hash, } // Resolve the transaction; if it doesn't exist, return nil. t, err := tx.resolve(ctx) @@ -1284,8 +1313,8 @@ func (r *Resolver) Logs(ctx context.Context, args struct{ Filter FilterCriteria topics = *args.Filter.Topics } // Construct the range filter - filter := filters.NewRangeFilter(filters.Backend(r.backend), begin, end, addresses, topics) - return runFilter(ctx, r.backend, filter) + filter := r.filterSystem.NewRangeFilter(begin, end, addresses, topics) + return runFilter(ctx, r, filter) } func (r *Resolver) GasPrice(ctx context.Context) (hexutil.Big, error) { diff --git a/graphql/graphql_test.go b/graphql/graphql_test.go index 2768026b5197e..491c731521138 100644 --- a/graphql/graphql_test.go +++ b/graphql/graphql_test.go @@ -17,6 +17,8 @@ package graphql import ( + "context" + "encoding/json" "fmt" "io" "math/big" @@ -33,6 +35,7 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/eth/ethconfig" + "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/params" @@ -45,20 +48,26 @@ func TestBuildSchema(t *testing.T) { conf := node.DefaultConfig conf.DataDir = ddir stack, err := node.New(&conf) - defer stack.Close() if err != nil { t.Fatalf("could not create new node: %v", err) } + defer stack.Close() // Make sure the schema can be parsed and matched up to the object model. - if err := newHandler(stack, nil, []string{}, []string{}); err != nil { + if _, err := newHandler(stack, nil, nil, []string{}, []string{}); err != nil { t.Errorf("Could not construct GraphQL handler: %v", err) } } // Tests that a graphQL request is successfully handled when graphql is enabled on the specified endpoint func TestGraphQLBlockSerialization(t *testing.T) { - stack := createNode(t, true, false) + stack := createNode(t) defer stack.Close() + genesis := &core.Genesis{ + Config: params.AllEthashProtocolChanges, + GasLimit: 11500000, + Difficulty: big.NewInt(1048576), + } + newGQLService(t, stack, genesis, 10, func(i int, gen *core.BlockGen) {}) // start node if err := stack.Start(); err != nil { t.Fatalf("could not start node: %v", err) @@ -160,8 +169,55 @@ func TestGraphQLBlockSerialization(t *testing.T) { } func TestGraphQLBlockSerializationEIP2718(t *testing.T) { - stack := createNode(t, true, true) + // Account for signing txes + var ( + key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + address = crypto.PubkeyToAddress(key.PublicKey) + funds = big.NewInt(1000000000000000) + dad = common.HexToAddress("0x0000000000000000000000000000000000000dad") + ) + stack := createNode(t) defer stack.Close() + genesis := &core.Genesis{ + Config: params.AllEthashProtocolChanges, + GasLimit: 11500000, + Difficulty: big.NewInt(1048576), + Alloc: core.GenesisAlloc{ + address: {Balance: funds}, + // The address 0xdad sloads 0x00 and 0x01 + dad: { + Code: []byte{byte(vm.PC), byte(vm.PC), byte(vm.SLOAD), byte(vm.SLOAD)}, + Nonce: 0, + Balance: big.NewInt(0), + }, + }, + BaseFee: big.NewInt(params.InitialBaseFee), + } + signer := types.LatestSigner(genesis.Config) + newGQLService(t, stack, genesis, 1, func(i int, gen *core.BlockGen) { + gen.SetCoinbase(common.Address{1}) + tx, _ := types.SignNewTx(key, signer, &types.LegacyTx{ + Nonce: uint64(0), + To: &dad, + Value: big.NewInt(100), + Gas: 50000, + GasPrice: big.NewInt(params.InitialBaseFee), + }) + gen.AddTx(tx) + tx, _ = types.SignNewTx(key, signer, &types.AccessListTx{ + ChainID: genesis.Config.ChainID, + Nonce: uint64(1), + To: &dad, + Gas: 30000, + GasPrice: big.NewInt(params.InitialBaseFee), + Value: big.NewInt(50), + AccessList: types.AccessList{{ + Address: dad, + StorageKeys: []common.Hash{{0}}, + }}, + }) + gen.AddTx(tx) + }) // start node if err := stack.Start(); err != nil { t.Fatalf("could not start node: %v", err) @@ -197,7 +253,7 @@ func TestGraphQLBlockSerializationEIP2718(t *testing.T) { // Tests that a graphQL request is not handled successfully when graphql is not enabled on the specified endpoint func TestGraphQLHTTPOnSamePort_GQLRequest_Unsuccessful(t *testing.T) { - stack := createNode(t, false, false) + stack := createNode(t) defer stack.Close() if err := stack.Start(); err != nil { t.Fatalf("could not start node: %v", err) @@ -211,7 +267,59 @@ func TestGraphQLHTTPOnSamePort_GQLRequest_Unsuccessful(t *testing.T) { assert.Equal(t, http.StatusNotFound, resp.StatusCode) } -func createNode(t *testing.T, gqlEnabled bool, txEnabled bool) *node.Node { +func TestGraphQLTransactionLogs(t *testing.T) { + var ( + key, _ = crypto.GenerateKey() + addr = crypto.PubkeyToAddress(key.PublicKey) + dadStr = "0x0000000000000000000000000000000000000dad" + dad = common.HexToAddress(dadStr) + genesis = &core.Genesis{ + Config: params.AllEthashProtocolChanges, + GasLimit: 11500000, + Difficulty: big.NewInt(1048576), + Alloc: core.GenesisAlloc{ + addr: {Balance: big.NewInt(params.Ether)}, + dad: { + // LOG0(0, 0), LOG0(0, 0), RETURN(0, 0) + Code: common.Hex2Bytes("60006000a060006000a060006000f3"), + Nonce: 0, + Balance: big.NewInt(0), + }, + }, + } + signer = types.LatestSigner(genesis.Config) + stack = createNode(t) + ) + defer stack.Close() + + handler := newGQLService(t, stack, genesis, 1, func(i int, gen *core.BlockGen) { + tx, _ := types.SignNewTx(key, signer, &types.LegacyTx{To: &dad, Gas: 100000, GasPrice: big.NewInt(params.InitialBaseFee)}) + gen.AddTx(tx) + tx, _ = types.SignNewTx(key, signer, &types.LegacyTx{To: &dad, Nonce: 1, Gas: 100000, GasPrice: big.NewInt(params.InitialBaseFee)}) + gen.AddTx(tx) + tx, _ = types.SignNewTx(key, signer, &types.LegacyTx{To: &dad, Nonce: 2, Gas: 100000, GasPrice: big.NewInt(params.InitialBaseFee)}) + gen.AddTx(tx) + }) + // start node + if err := stack.Start(); err != nil { + t.Fatalf("could not start node: %v", err) + } + query := `{block { transactions { logs { account { address } } } } }` + res := handler.Schema.Exec(context.Background(), query, "", map[string]interface{}{}) + if res.Errors != nil { + t.Fatalf("graphql query failed: %v", res.Errors) + } + have, err := json.Marshal(res.Data) + if err != nil { + t.Fatalf("failed to encode graphql response: %s", err) + } + want := fmt.Sprintf(`{"block":{"transactions":[{"logs":[{"account":{"address":"%s"}},{"account":{"address":"%s"}}]},{"logs":[{"account":{"address":"%s"}},{"account":{"address":"%s"}}]},{"logs":[{"account":{"address":"%s"}},{"account":{"address":"%s"}}]}]}}`, dadStr, dadStr, dadStr, dadStr, dadStr, dadStr) + if string(have) != want { + t.Errorf("response unmatch. expected %s, got %s", want, have) + } +} + +func createNode(t *testing.T) *node.Node { stack, err := node.New(&node.Config{ HTTPHost: "127.0.0.1", HTTPPort: 0, @@ -221,25 +329,12 @@ func createNode(t *testing.T, gqlEnabled bool, txEnabled bool) *node.Node { if err != nil { t.Fatalf("could not create node: %v", err) } - if !gqlEnabled { - return stack - } - if !txEnabled { - createGQLService(t, stack) - } else { - createGQLServiceWithTransactions(t, stack) - } return stack } -func createGQLService(t *testing.T, stack *node.Node) { - // create backend +func newGQLService(t *testing.T, stack *node.Node, gspec *core.Genesis, genBlocks int, genfunc func(i int, gen *core.BlockGen)) *handler { ethConf := ðconfig.Config{ - Genesis: &core.Genesis{ - Config: params.AllEthashProtocolChanges, - GasLimit: 11500000, - Difficulty: big.NewInt(1048576), - }, + Genesis: gspec, Ethash: ethash.Config{ PowMode: ethash.ModeFake, }, @@ -257,99 +352,16 @@ func createGQLService(t *testing.T, stack *node.Node) { } // Create some blocks and import them chain, _ := core.GenerateChain(params.AllEthashProtocolChanges, ethBackend.BlockChain().Genesis(), - ethash.NewFaker(), ethBackend.ChainDb(), 10, func(i int, gen *core.BlockGen) {}) - _, err = ethBackend.BlockChain().InsertChain(chain) - if err != nil { - t.Fatalf("could not create import blocks: %v", err) - } - // create gql service - err = New(stack, ethBackend.APIBackend, []string{}, []string{}) - if err != nil { - t.Fatalf("could not create graphql service: %v", err) - } -} - -func createGQLServiceWithTransactions(t *testing.T, stack *node.Node) { - // create backend - key, _ := crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - address := crypto.PubkeyToAddress(key.PublicKey) - funds := big.NewInt(1000000000000000) - dad := common.HexToAddress("0x0000000000000000000000000000000000000dad") - - ethConf := ðconfig.Config{ - Genesis: &core.Genesis{ - Config: params.AllEthashProtocolChanges, - GasLimit: 11500000, - Difficulty: big.NewInt(1048576), - Alloc: core.GenesisAlloc{ - address: {Balance: funds}, - // The address 0xdad sloads 0x00 and 0x01 - dad: { - Code: []byte{ - byte(vm.PC), - byte(vm.PC), - byte(vm.SLOAD), - byte(vm.SLOAD), - }, - Nonce: 0, - Balance: big.NewInt(0), - }, - }, - BaseFee: big.NewInt(params.InitialBaseFee), - }, - Ethash: ethash.Config{ - PowMode: ethash.ModeFake, - }, - NetworkId: 1337, - TrieCleanCache: 5, - TrieCleanCacheJournal: "triecache", - TrieCleanCacheRejournal: 60 * time.Minute, - TrieDirtyCache: 5, - TrieTimeout: 60 * time.Minute, - SnapshotCache: 5, - } - - ethBackend, err := eth.New(stack, ethConf) - if err != nil { - t.Fatalf("could not create eth backend: %v", err) - } - signer := types.LatestSigner(ethConf.Genesis.Config) - - legacyTx, _ := types.SignNewTx(key, signer, &types.LegacyTx{ - Nonce: uint64(0), - To: &dad, - Value: big.NewInt(100), - Gas: 50000, - GasPrice: big.NewInt(params.InitialBaseFee), - }) - envelopTx, _ := types.SignNewTx(key, signer, &types.AccessListTx{ - ChainID: ethConf.Genesis.Config.ChainID, - Nonce: uint64(1), - To: &dad, - Gas: 30000, - GasPrice: big.NewInt(params.InitialBaseFee), - Value: big.NewInt(50), - AccessList: types.AccessList{{ - Address: dad, - StorageKeys: []common.Hash{{0}}, - }}, - }) - - // Create some blocks and import them - chain, _ := core.GenerateChain(params.AllEthashProtocolChanges, ethBackend.BlockChain().Genesis(), - ethash.NewFaker(), ethBackend.ChainDb(), 1, func(i int, b *core.BlockGen) { - b.SetCoinbase(common.Address{1}) - b.AddTx(legacyTx) - b.AddTx(envelopTx) - }) - + ethash.NewFaker(), ethBackend.ChainDb(), genBlocks, genfunc) _, err = ethBackend.BlockChain().InsertChain(chain) if err != nil { t.Fatalf("could not create import blocks: %v", err) } - // create gql service - err = New(stack, ethBackend.APIBackend, []string{}, []string{}) + // Set up handler + filterSystem := filters.NewFilterSystem(ethBackend.APIBackend, filters.Config{}) + handler, err := newHandler(stack, ethBackend.APIBackend, filterSystem, []string{}, []string{}) if err != nil { t.Fatalf("could not create graphql service: %v", err) } + return handler } diff --git a/graphql/service.go b/graphql/service.go index 29d98ad746836..6f6e58335991c 100644 --- a/graphql/service.go +++ b/graphql/service.go @@ -20,6 +20,7 @@ import ( "encoding/json" "net/http" + "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/internal/ethapi" "github.com/ethereum/go-ethereum/node" "github.com/graph-gophers/graphql-go" @@ -52,26 +53,22 @@ func (h handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") w.Write(responseJSON) - } // New constructs a new GraphQL service instance. -func New(stack *node.Node, backend ethapi.Backend, cors, vhosts []string) error { - if backend == nil { - panic("missing backend") - } - // check if http server with given endpoint exists and enable graphQL on it - return newHandler(stack, backend, cors, vhosts) +func New(stack *node.Node, backend ethapi.Backend, filterSystem *filters.FilterSystem, cors, vhosts []string) error { + _, err := newHandler(stack, backend, filterSystem, cors, vhosts) + return err } // newHandler returns a new `http.Handler` that will answer GraphQL queries. // It additionally exports an interactive query browser on the / endpoint. -func newHandler(stack *node.Node, backend ethapi.Backend, cors, vhosts []string) error { - q := Resolver{backend} +func newHandler(stack *node.Node, backend ethapi.Backend, filterSystem *filters.FilterSystem, cors, vhosts []string) (*handler, error) { + q := Resolver{backend, filterSystem} s, err := graphql.ParseSchema(schema, &q) if err != nil { - return err + return nil, err } h := handler{Schema: s} handler := node.NewHTTPHandlerStack(h, cors, vhosts, nil) @@ -80,5 +77,5 @@ func newHandler(stack *node.Node, backend ethapi.Backend, cors, vhosts []string) stack.RegisterHandler("GraphQL", "/graphql", handler) stack.RegisterHandler("GraphQL", "/graphql/", handler) - return nil + return &h, nil } diff --git a/interfaces.go b/interfaces.go index 76c1ef6908f25..eb9af60076e07 100644 --- a/interfaces.go +++ b/interfaces.go @@ -201,6 +201,15 @@ type GasPricer interface { SuggestGasPrice(ctx context.Context) (*big.Int, error) } +// FeeHistory provides recent fee market data that consumers can use to determine +// a reasonable maxPriorityFeePerGas value. +type FeeHistory struct { + OldestBlock *big.Int // block corresponding to first response value + Reward [][]*big.Int // list every txs priority fee per block + BaseFee []*big.Int // list of each block's base fee + GasUsedRatio []float64 // ratio of gas used out of the total available limit +} + // A PendingStateReader provides access to the pending state, which is the result of all // known executable transactions which have not yet been included in the blockchain. It is // commonly used to display the result of ’unconfirmed’ actions (e.g. wallet value diff --git a/internal/build/archive.go b/internal/build/archive.go index 8b3ac23d1d899..c16246070e8c8 100644 --- a/internal/build/archive.go +++ b/internal/build/archive.go @@ -25,6 +25,7 @@ import ( "os" "path/filepath" "strings" + "time" ) type Archive interface { @@ -159,6 +160,7 @@ func (a *TarballArchive) Directory(name string) error { Name: a.dir, Mode: 0755, Typeflag: tar.TypeDir, + ModTime: time.Now(), }) } diff --git a/internal/build/gotool.go b/internal/build/gotool.go index e644b5f69526d..08c8b2ef05fe8 100644 --- a/internal/build/gotool.go +++ b/internal/build/gotool.go @@ -85,7 +85,7 @@ func (g *GoToolchain) goTool(command string, args ...string) *exec.Cmd { if g.Root == "" { g.Root = runtime.GOROOT() } - tool := exec.Command(filepath.Join(g.Root, "bin", "go"), command) + tool := exec.Command(filepath.Join(g.Root, "bin", "go"), command) // nolint: gosec tool.Args = append(tool.Args, args...) tool.Env = append(tool.Env, "GOROOT="+g.Root) diff --git a/internal/build/util.go b/internal/build/util.go index 654349fac307c..9a721e9b83b16 100644 --- a/internal/build/util.go +++ b/internal/build/util.go @@ -29,6 +29,7 @@ import ( "os/exec" "path" "path/filepath" + "strconv" "strings" "text/template" "time" @@ -39,7 +40,7 @@ var DryRunFlag = flag.Bool("n", false, "dry run, don't execute commands") // MustRun executes the given command and exits the host process for // any error. func MustRun(cmd *exec.Cmd) { - fmt.Println(">>>", strings.Join(cmd.Args, " ")) + fmt.Println(">>>", printArgs(cmd.Args)) if !*DryRunFlag { cmd.Stderr = os.Stderr cmd.Stdout = os.Stdout @@ -49,6 +50,20 @@ func MustRun(cmd *exec.Cmd) { } } +func printArgs(args []string) string { + var s strings.Builder + for i, arg := range args { + if i > 0 { + s.WriteByte(' ') + } + if strings.IndexByte(arg, ' ') >= 0 { + arg = strconv.QuoteToASCII(arg) + } + s.WriteString(arg) + } + return s.String() +} + func MustRunCommand(cmd string, args ...string) { MustRun(exec.Command(cmd, args...)) } @@ -121,7 +136,7 @@ func UploadSFTP(identityFile, host, dir string, files []string) error { sftp.Args = append(sftp.Args, "-i", identityFile) } sftp.Args = append(sftp.Args, host) - fmt.Println(">>>", strings.Join(sftp.Args, " ")) + fmt.Println(">>>", printArgs(sftp.Args)) if *DryRunFlag { return nil } diff --git a/internal/cmdtest/test_cmd.go b/internal/cmdtest/test_cmd.go index b837c9c399ca9..fd7a4a8b7f4c2 100644 --- a/internal/cmdtest/test_cmd.go +++ b/internal/cmdtest/test_cmd.go @@ -83,7 +83,7 @@ func (tt *TestCmd) Run(name string, args ...string) { // InputLine writes the given text to the child's stdin. // This method can also be called from an expect template, e.g.: // -// geth.expect(`Passphrase: {{.InputLine "password"}}`) +// geth.expect(`Passphrase: {{.InputLine "password"}}`) func (tt *TestCmd) InputLine(s string) string { io.WriteString(tt.stdin, s+"\n") return "" diff --git a/internal/debug/flags.go b/internal/debug/flags.go index 3aa990adfb3ab..2082d60df51c0 100644 --- a/internal/debug/flags.go +++ b/internal/debug/flags.go @@ -20,75 +20,88 @@ import ( "fmt" "io" "net/http" - _ "net/http/pprof" + _ "net/http/pprof" // nolint: gosec "os" "runtime" + "github.com/ethereum/go-ethereum/internal/flags" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/metrics/exp" "github.com/fjl/memsize/memsizeui" "github.com/mattn/go-colorable" "github.com/mattn/go-isatty" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) var Memsize memsizeui.Handler var ( - verbosityFlag = cli.IntFlag{ - Name: "verbosity", - Usage: "Logging verbosity: 0=silent, 1=error, 2=warn, 3=info, 4=debug, 5=detail", - Value: 3, - } - vmoduleFlag = cli.StringFlag{ - Name: "vmodule", - Usage: "Per-module verbosity: comma-separated list of = (e.g. eth/*=5,p2p=4)", - Value: "", - } - logjsonFlag = cli.BoolFlag{ - Name: "log.json", - Usage: "Format logs with JSON", - } - backtraceAtFlag = cli.StringFlag{ - Name: "log.backtrace", - Usage: "Request a stack trace at a specific logging statement (e.g. \"block.go:271\")", - Value: "", - } - debugFlag = cli.BoolFlag{ - Name: "log.debug", - Usage: "Prepends log messages with call-site location (file and line number)", - } - pprofFlag = cli.BoolFlag{ - Name: "pprof", - Usage: "Enable the pprof HTTP server", - } - pprofPortFlag = cli.IntFlag{ - Name: "pprof.port", - Usage: "pprof HTTP server listening port", - Value: 6060, - } - pprofAddrFlag = cli.StringFlag{ - Name: "pprof.addr", - Usage: "pprof HTTP server listening interface", - Value: "127.0.0.1", - } - memprofilerateFlag = cli.IntFlag{ - Name: "pprof.memprofilerate", - Usage: "Turn on memory profiling with the given rate", - Value: runtime.MemProfileRate, - } - blockprofilerateFlag = cli.IntFlag{ - Name: "pprof.blockprofilerate", - Usage: "Turn on block profiling with the given rate", - } - cpuprofileFlag = cli.StringFlag{ - Name: "pprof.cpuprofile", - Usage: "Write CPU profile to the given file", - } - traceFlag = cli.StringFlag{ - Name: "trace", - Usage: "Write execution trace to the given file", + verbosityFlag = &cli.IntFlag{ + Name: "verbosity", + Usage: "Logging verbosity: 0=silent, 1=error, 2=warn, 3=info, 4=debug, 5=detail", + Value: 3, + Category: flags.LoggingCategory, + } + vmoduleFlag = &cli.StringFlag{ + Name: "vmodule", + Usage: "Per-module verbosity: comma-separated list of = (e.g. eth/*=5,p2p=4)", + Value: "", + Category: flags.LoggingCategory, + } + logjsonFlag = &cli.BoolFlag{ + Name: "log.json", + Usage: "Format logs with JSON", + Category: flags.LoggingCategory, + } + backtraceAtFlag = &cli.StringFlag{ + Name: "log.backtrace", + Usage: "Request a stack trace at a specific logging statement (e.g. \"block.go:271\")", + Value: "", + Category: flags.LoggingCategory, + } + debugFlag = &cli.BoolFlag{ + Name: "log.debug", + Usage: "Prepends log messages with call-site location (file and line number)", + Category: flags.LoggingCategory, + } + pprofFlag = &cli.BoolFlag{ + Name: "pprof", + Usage: "Enable the pprof HTTP server", + Category: flags.LoggingCategory, + } + pprofPortFlag = &cli.IntFlag{ + Name: "pprof.port", + Usage: "pprof HTTP server listening port", + Value: 6060, + Category: flags.LoggingCategory, + } + pprofAddrFlag = &cli.StringFlag{ + Name: "pprof.addr", + Usage: "pprof HTTP server listening interface", + Value: "127.0.0.1", + Category: flags.LoggingCategory, + } + memprofilerateFlag = &cli.IntFlag{ + Name: "pprof.memprofilerate", + Usage: "Turn on memory profiling with the given rate", + Value: runtime.MemProfileRate, + Category: flags.LoggingCategory, + } + blockprofilerateFlag = &cli.IntFlag{ + Name: "pprof.blockprofilerate", + Usage: "Turn on block profiling with the given rate", + Category: flags.LoggingCategory, + } + cpuprofileFlag = &cli.StringFlag{ + Name: "pprof.cpuprofile", + Usage: "Write CPU profile to the given file", + Category: flags.LoggingCategory, + } + traceFlag = &cli.StringFlag{ + Name: "trace", + Usage: "Write execution trace to the given file", + Category: flags.LoggingCategory, } ) @@ -121,7 +134,7 @@ func init() { func Setup(ctx *cli.Context) error { var ostream log.Handler output := io.Writer(os.Stderr) - if ctx.GlobalBool(logjsonFlag.Name) { + if ctx.Bool(logjsonFlag.Name) { ostream = log.StreamHandler(output, log.JSONFormat()) } else { usecolor := (isatty.IsTerminal(os.Stderr.Fd()) || isatty.IsCygwinTerminal(os.Stderr.Fd())) && os.Getenv("TERM") != "dumb" @@ -133,53 +146,53 @@ func Setup(ctx *cli.Context) error { glogger.SetHandler(ostream) // logging - verbosity := ctx.GlobalInt(verbosityFlag.Name) + verbosity := ctx.Int(verbosityFlag.Name) glogger.Verbosity(log.Lvl(verbosity)) - vmodule := ctx.GlobalString(vmoduleFlag.Name) + vmodule := ctx.String(vmoduleFlag.Name) glogger.Vmodule(vmodule) - debug := ctx.GlobalBool(debugFlag.Name) - if ctx.GlobalIsSet(debugFlag.Name) { - debug = ctx.GlobalBool(debugFlag.Name) + debug := ctx.Bool(debugFlag.Name) + if ctx.IsSet(debugFlag.Name) { + debug = ctx.Bool(debugFlag.Name) } log.PrintOrigins(debug) - backtrace := ctx.GlobalString(backtraceAtFlag.Name) + backtrace := ctx.String(backtraceAtFlag.Name) glogger.BacktraceAt(backtrace) log.Root().SetHandler(glogger) // profiling, tracing runtime.MemProfileRate = memprofilerateFlag.Value - if ctx.GlobalIsSet(memprofilerateFlag.Name) { - runtime.MemProfileRate = ctx.GlobalInt(memprofilerateFlag.Name) + if ctx.IsSet(memprofilerateFlag.Name) { + runtime.MemProfileRate = ctx.Int(memprofilerateFlag.Name) } - blockProfileRate := ctx.GlobalInt(blockprofilerateFlag.Name) + blockProfileRate := ctx.Int(blockprofilerateFlag.Name) Handler.SetBlockProfileRate(blockProfileRate) - if traceFile := ctx.GlobalString(traceFlag.Name); traceFile != "" { + if traceFile := ctx.String(traceFlag.Name); traceFile != "" { if err := Handler.StartGoTrace(traceFile); err != nil { return err } } - if cpuFile := ctx.GlobalString(cpuprofileFlag.Name); cpuFile != "" { + if cpuFile := ctx.String(cpuprofileFlag.Name); cpuFile != "" { if err := Handler.StartCPUProfile(cpuFile); err != nil { return err } } // pprof server - if ctx.GlobalBool(pprofFlag.Name) { - listenHost := ctx.GlobalString(pprofAddrFlag.Name) + if ctx.Bool(pprofFlag.Name) { + listenHost := ctx.String(pprofAddrFlag.Name) - port := ctx.GlobalInt(pprofPortFlag.Name) + port := ctx.Int(pprofPortFlag.Name) address := fmt.Sprintf("%s:%d", listenHost, port) // This context value ("metrics.addr") represents the utils.MetricsHTTPFlag.Name. // It cannot be imported because it will cause a cyclical dependency. - StartPProf(address, !ctx.GlobalIsSet("metrics.addr")) + StartPProf(address, !ctx.IsSet("metrics.addr")) } return nil } diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 7c422e642d707..64b389612aba3 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -18,6 +18,7 @@ package ethapi import ( "context" + "encoding/hex" "errors" "fmt" "math/big" @@ -48,19 +49,18 @@ import ( "github.com/tyler-smith/go-bip39" ) -// PublicEthereumAPI provides an API to access Ethereum related information. -// It offers only methods that operate on public data that is freely available to anyone. -type PublicEthereumAPI struct { +// EthereumAPI provides an API to access Ethereum related information. +type EthereumAPI struct { b Backend } -// NewPublicEthereumAPI creates a new Ethereum protocol API. -func NewPublicEthereumAPI(b Backend) *PublicEthereumAPI { - return &PublicEthereumAPI{b} +// NewEthereumAPI creates a new Ethereum protocol API. +func NewEthereumAPI(b Backend) *EthereumAPI { + return &EthereumAPI{b} } // GasPrice returns a suggestion for a gas price for legacy transactions. -func (s *PublicEthereumAPI) GasPrice(ctx context.Context) (*hexutil.Big, error) { +func (s *EthereumAPI) GasPrice(ctx context.Context) (*hexutil.Big, error) { tipcap, err := s.b.SuggestGasTipCap(ctx) if err != nil { return nil, err @@ -72,7 +72,7 @@ func (s *PublicEthereumAPI) GasPrice(ctx context.Context) (*hexutil.Big, error) } // MaxPriorityFeePerGas returns a suggestion for a gas tip cap for dynamic fee transactions. -func (s *PublicEthereumAPI) MaxPriorityFeePerGas(ctx context.Context) (*hexutil.Big, error) { +func (s *EthereumAPI) MaxPriorityFeePerGas(ctx context.Context) (*hexutil.Big, error) { tipcap, err := s.b.SuggestGasTipCap(ctx) if err != nil { return nil, err @@ -87,7 +87,8 @@ type feeHistoryResult struct { GasUsedRatio []float64 `json:"gasUsedRatio"` } -func (s *PublicEthereumAPI) FeeHistory(ctx context.Context, blockCount rpc.DecimalOrHex, lastBlock rpc.BlockNumber, rewardPercentiles []float64) (*feeHistoryResult, error) { +// FeeHistory returns the fee market history. +func (s *EthereumAPI) FeeHistory(ctx context.Context, blockCount rpc.DecimalOrHex, lastBlock rpc.BlockNumber, rewardPercentiles []float64) (*feeHistoryResult, error) { oldest, reward, baseFee, gasUsed, err := s.b.FeeHistory(ctx, int(blockCount), lastBlock, rewardPercentiles) if err != nil { return nil, err @@ -121,7 +122,7 @@ func (s *PublicEthereumAPI) FeeHistory(ctx context.Context, blockCount rpc.Decim // - highestBlock: block number of the highest block header this node has received from peers // - pulledStates: number of state entries processed until now // - knownStates: number of known state entries that still need to be pulled -func (s *PublicEthereumAPI) Syncing() (interface{}, error) { +func (s *EthereumAPI) Syncing() (interface{}, error) { progress := s.b.SyncProgress() // Return not syncing if the synchronisation already completed @@ -148,18 +149,18 @@ func (s *PublicEthereumAPI) Syncing() (interface{}, error) { }, nil } -// PublicTxPoolAPI offers and API for the transaction pool. It only operates on data that is non confidential. -type PublicTxPoolAPI struct { +// TxPoolAPI offers and API for the transaction pool. It only operates on data that is non confidential. +type TxPoolAPI struct { b Backend } -// NewPublicTxPoolAPI creates a new tx pool service that gives information about the transaction pool. -func NewPublicTxPoolAPI(b Backend) *PublicTxPoolAPI { - return &PublicTxPoolAPI{b} +// NewTxPoolAPI creates a new tx pool service that gives information about the transaction pool. +func NewTxPoolAPI(b Backend) *TxPoolAPI { + return &TxPoolAPI{b} } // Content returns the transactions contained within the transaction pool. -func (s *PublicTxPoolAPI) Content() map[string]map[string]map[string]*RPCTransaction { +func (s *TxPoolAPI) Content() map[string]map[string]map[string]*RPCTransaction { content := map[string]map[string]map[string]*RPCTransaction{ "pending": make(map[string]map[string]*RPCTransaction), "queued": make(map[string]map[string]*RPCTransaction), @@ -186,7 +187,7 @@ func (s *PublicTxPoolAPI) Content() map[string]map[string]map[string]*RPCTransac } // ContentFrom returns the transactions contained within the transaction pool. -func (s *PublicTxPoolAPI) ContentFrom(addr common.Address) map[string]map[string]*RPCTransaction { +func (s *TxPoolAPI) ContentFrom(addr common.Address) map[string]map[string]*RPCTransaction { content := make(map[string]map[string]*RPCTransaction, 2) pending, queue := s.b.TxPoolContentFrom(addr) curHeader := s.b.CurrentHeader() @@ -209,7 +210,7 @@ func (s *PublicTxPoolAPI) ContentFrom(addr common.Address) map[string]map[string } // Status returns the number of pending and queued transaction in the pool. -func (s *PublicTxPoolAPI) Status() map[string]hexutil.Uint { +func (s *TxPoolAPI) Status() map[string]hexutil.Uint { pending, queue := s.b.Stats() return map[string]hexutil.Uint{ "pending": hexutil.Uint(pending), @@ -219,7 +220,7 @@ func (s *PublicTxPoolAPI) Status() map[string]hexutil.Uint { // Inspect retrieves the content of the transaction pool and flattens it into an // easily inspectable list. -func (s *PublicTxPoolAPI) Inspect() map[string]map[string]map[string]string { +func (s *TxPoolAPI) Inspect() map[string]map[string]map[string]string { content := map[string]map[string]map[string]string{ "pending": make(map[string]map[string]string), "queued": make(map[string]map[string]string), @@ -252,34 +253,34 @@ func (s *PublicTxPoolAPI) Inspect() map[string]map[string]map[string]string { return content } -// PublicAccountAPI provides an API to access accounts managed by this node. +// EthereumAccountAPI provides an API to access accounts managed by this node. // It offers only methods that can retrieve accounts. -type PublicAccountAPI struct { +type EthereumAccountAPI struct { am *accounts.Manager } -// NewPublicAccountAPI creates a new PublicAccountAPI. -func NewPublicAccountAPI(am *accounts.Manager) *PublicAccountAPI { - return &PublicAccountAPI{am: am} +// NewEthereumAccountAPI creates a new EthereumAccountAPI. +func NewEthereumAccountAPI(am *accounts.Manager) *EthereumAccountAPI { + return &EthereumAccountAPI{am: am} } -// Accounts returns the collection of accounts this node manages -func (s *PublicAccountAPI) Accounts() []common.Address { +// Accounts returns the collection of accounts this node manages. +func (s *EthereumAccountAPI) Accounts() []common.Address { return s.am.Accounts() } -// PrivateAccountAPI provides an API to access accounts managed by this node. +// PersonalAccountAPI provides an API to access accounts managed by this node. // It offers methods to create, (un)lock en list accounts. Some methods accept // passwords and are therefore considered private by default. -type PrivateAccountAPI struct { +type PersonalAccountAPI struct { am *accounts.Manager nonceLock *AddrLocker b Backend } -// NewPrivateAccountAPI create a new PrivateAccountAPI. -func NewPrivateAccountAPI(b Backend, nonceLock *AddrLocker) *PrivateAccountAPI { - return &PrivateAccountAPI{ +// NewPersonalAccountAPI create a new PersonalAccountAPI. +func NewPersonalAccountAPI(b Backend, nonceLock *AddrLocker) *PersonalAccountAPI { + return &PersonalAccountAPI{ am: b.AccountManager(), nonceLock: nonceLock, b: b, @@ -287,7 +288,7 @@ func NewPrivateAccountAPI(b Backend, nonceLock *AddrLocker) *PrivateAccountAPI { } // ListAccounts will return a list of addresses for accounts this node manages. -func (s *PrivateAccountAPI) ListAccounts() []common.Address { +func (s *PersonalAccountAPI) ListAccounts() []common.Address { return s.am.Accounts() } @@ -301,7 +302,7 @@ type rawWallet struct { } // ListWallets will return a list of wallets this node manages. -func (s *PrivateAccountAPI) ListWallets() []rawWallet { +func (s *PersonalAccountAPI) ListWallets() []rawWallet { wallets := make([]rawWallet, 0) // return [] instead of nil if empty for _, wallet := range s.am.Wallets() { status, failure := wallet.Status() @@ -323,7 +324,7 @@ func (s *PrivateAccountAPI) ListWallets() []rawWallet { // connection and attempting to authenticate via the provided passphrase. Note, // the method may return an extra challenge requiring a second open (e.g. the // Trezor PIN matrix challenge). -func (s *PrivateAccountAPI) OpenWallet(url string, passphrase *string) error { +func (s *PersonalAccountAPI) OpenWallet(url string, passphrase *string) error { wallet, err := s.am.Wallet(url) if err != nil { return err @@ -337,7 +338,7 @@ func (s *PrivateAccountAPI) OpenWallet(url string, passphrase *string) error { // DeriveAccount requests a HD wallet to derive a new account, optionally pinning // it for later reuse. -func (s *PrivateAccountAPI) DeriveAccount(url string, path string, pin *bool) (accounts.Account, error) { +func (s *PersonalAccountAPI) DeriveAccount(url string, path string, pin *bool) (accounts.Account, error) { wallet, err := s.am.Wallet(url) if err != nil { return accounts.Account{}, err @@ -353,7 +354,7 @@ func (s *PrivateAccountAPI) DeriveAccount(url string, path string, pin *bool) (a } // NewAccount will create a new account and returns the address for the new account. -func (s *PrivateAccountAPI) NewAccount(password string) (common.Address, error) { +func (s *PersonalAccountAPI) NewAccount(password string) (common.Address, error) { ks, err := fetchKeystore(s.am) if err != nil { return common.Address{}, err @@ -378,7 +379,7 @@ func fetchKeystore(am *accounts.Manager) (*keystore.KeyStore, error) { // ImportRawKey stores the given hex encoded ECDSA key into the key directory, // encrypting it with the passphrase. -func (s *PrivateAccountAPI) ImportRawKey(privkey string, password string) (common.Address, error) { +func (s *PersonalAccountAPI) ImportRawKey(privkey string, password string) (common.Address, error) { key, err := crypto.HexToECDSA(privkey) if err != nil { return common.Address{}, err @@ -394,7 +395,7 @@ func (s *PrivateAccountAPI) ImportRawKey(privkey string, password string) (commo // UnlockAccount will unlock the account associated with the given address with // the given password for duration seconds. If duration is nil it will use a // default of 300 seconds. It returns an indication if the account was unlocked. -func (s *PrivateAccountAPI) UnlockAccount(ctx context.Context, addr common.Address, password string, duration *uint64) (bool, error) { +func (s *PersonalAccountAPI) UnlockAccount(ctx context.Context, addr common.Address, password string, duration *uint64) (bool, error) { // When the API is exposed by external RPC(http, ws etc), unless the user // explicitly specifies to allow the insecure account unlocking, otherwise // it is disabled. @@ -423,7 +424,7 @@ func (s *PrivateAccountAPI) UnlockAccount(ctx context.Context, addr common.Addre } // LockAccount will lock the account associated with the given address when it's unlocked. -func (s *PrivateAccountAPI) LockAccount(addr common.Address) bool { +func (s *PersonalAccountAPI) LockAccount(addr common.Address) bool { if ks, err := fetchKeystore(s.am); err == nil { return ks.Lock(addr) == nil } @@ -433,7 +434,7 @@ func (s *PrivateAccountAPI) LockAccount(addr common.Address) bool { // signTransaction sets defaults and signs the given transaction // NOTE: the caller needs to ensure that the nonceLock is held, if applicable, // and release it after the transaction has been submitted to the tx pool -func (s *PrivateAccountAPI) signTransaction(ctx context.Context, args *TransactionArgs, passwd string) (*types.Transaction, error) { +func (s *PersonalAccountAPI) signTransaction(ctx context.Context, args *TransactionArgs, passwd string) (*types.Transaction, error) { // Look up the wallet containing the requested signer account := accounts.Account{Address: args.from()} wallet, err := s.am.Find(account) @@ -453,7 +454,7 @@ func (s *PrivateAccountAPI) signTransaction(ctx context.Context, args *Transacti // SendTransaction will create a transaction from the given arguments and // tries to sign it with the key associated with args.From. If the given // passwd isn't able to decrypt the key it fails. -func (s *PrivateAccountAPI) SendTransaction(ctx context.Context, args TransactionArgs, passwd string) (common.Hash, error) { +func (s *PersonalAccountAPI) SendTransaction(ctx context.Context, args TransactionArgs, passwd string) (common.Hash, error) { if args.Nonce == nil { // Hold the addresse's mutex around signing to prevent concurrent assignment of // the same nonce to multiple accounts. @@ -472,7 +473,7 @@ func (s *PrivateAccountAPI) SendTransaction(ctx context.Context, args Transactio // tries to sign it with the key associated with args.From. If the given passwd isn't // able to decrypt the key it fails. The transaction is returned in RLP-form, not broadcast // to other nodes -func (s *PrivateAccountAPI) SignTransaction(ctx context.Context, args TransactionArgs, passwd string) (*SignTransactionResult, error) { +func (s *PersonalAccountAPI) SignTransaction(ctx context.Context, args TransactionArgs, passwd string) (*SignTransactionResult, error) { // No need to obtain the noncelock mutex, since we won't be sending this // tx into the transaction pool, but right back to the user if args.From == nil { @@ -505,7 +506,7 @@ func (s *PrivateAccountAPI) SignTransaction(ctx context.Context, args Transactio } // Sign calculates an Ethereum ECDSA signature for: -// keccack256("\x19Ethereum Signed Message:\n" + len(message) + message)) +// keccak256("\x19Ethereum Signed Message:\n" + len(message) + message)) // // Note, the produced signature conforms to the secp256k1 curve R, S and V values, // where the V value will be 27 or 28 for legacy reasons. @@ -513,7 +514,7 @@ func (s *PrivateAccountAPI) SignTransaction(ctx context.Context, args Transactio // The key used to calculate the signature is decrypted with the given password. // // https://github.com/ethereum/go-ethereum/wiki/Management-APIs#personal_sign -func (s *PrivateAccountAPI) Sign(ctx context.Context, data hexutil.Bytes, addr common.Address, passwd string) (hexutil.Bytes, error) { +func (s *PersonalAccountAPI) Sign(ctx context.Context, data hexutil.Bytes, addr common.Address, passwd string) (hexutil.Bytes, error) { // Look up the wallet containing the requested signer account := accounts.Account{Address: addr} @@ -541,7 +542,7 @@ func (s *PrivateAccountAPI) Sign(ctx context.Context, data hexutil.Bytes, addr c // the V value must be 27 or 28 for legacy reasons. // // https://github.com/ethereum/go-ethereum/wiki/Management-APIs#personal_ecRecover -func (s *PrivateAccountAPI) EcRecover(ctx context.Context, data, sig hexutil.Bytes) (common.Address, error) { +func (s *PersonalAccountAPI) EcRecover(ctx context.Context, data, sig hexutil.Bytes) (common.Address, error) { if len(sig) != crypto.SignatureLength { return common.Address{}, fmt.Errorf("signature must be %d bytes long", crypto.SignatureLength) } @@ -557,14 +558,8 @@ func (s *PrivateAccountAPI) EcRecover(ctx context.Context, data, sig hexutil.Byt return crypto.PubkeyToAddress(*rpk), nil } -// SignAndSendTransaction was renamed to SendTransaction. This method is deprecated -// and will be removed in the future. It primary goal is to give clients time to update. -func (s *PrivateAccountAPI) SignAndSendTransaction(ctx context.Context, args TransactionArgs, passwd string) (common.Hash, error) { - return s.SendTransaction(ctx, args, passwd) -} - // InitializeWallet initializes a new wallet at the provided URL, by generating and returning a new private key. -func (s *PrivateAccountAPI) InitializeWallet(ctx context.Context, url string) (string, error) { +func (s *PersonalAccountAPI) InitializeWallet(ctx context.Context, url string) (string, error) { wallet, err := s.am.Wallet(url) if err != nil { return "", err @@ -591,7 +586,7 @@ func (s *PrivateAccountAPI) InitializeWallet(ctx context.Context, url string) (s } // Unpair deletes a pairing between wallet and geth. -func (s *PrivateAccountAPI) Unpair(ctx context.Context, url string, pin string) error { +func (s *PersonalAccountAPI) Unpair(ctx context.Context, url string, pin string) error { wallet, err := s.am.Wallet(url) if err != nil { return err @@ -605,28 +600,28 @@ func (s *PrivateAccountAPI) Unpair(ctx context.Context, url string, pin string) } } -// PublicBlockChainAPI provides an API to access the Ethereum blockchain. -// It offers only methods that operate on public data that is freely available to anyone. -type PublicBlockChainAPI struct { +// BlockChainAPI provides an API to access Ethereum blockchain data. +type BlockChainAPI struct { b Backend } -// NewPublicBlockChainAPI creates a new Ethereum blockchain API. -func NewPublicBlockChainAPI(b Backend) *PublicBlockChainAPI { - return &PublicBlockChainAPI{b} +// NewBlockChainAPI creates a new Ethereum blockchain API. +func NewBlockChainAPI(b Backend) *BlockChainAPI { + return &BlockChainAPI{b} } -// ChainId is the EIP-155 replay-protection chain id for the current ethereum chain config. -func (api *PublicBlockChainAPI) ChainId() (*hexutil.Big, error) { - // if current block is at or past the EIP-155 replay-protection fork block, return chainID from config - if config := api.b.ChainConfig(); config.IsEIP155(api.b.CurrentBlock().Number()) { - return (*hexutil.Big)(config.ChainID), nil - } - return nil, fmt.Errorf("chain not synced beyond EIP-155 replay-protection fork block") +// ChainId is the EIP-155 replay-protection chain id for the current Ethereum chain config. +// +// Note, this method does not conform to EIP-695 because the configured chain ID is always +// returned, regardless of the current head block. We used to return an error when the chain +// wasn't synced up to a block where EIP-155 is enabled, but this behavior caused issues +// in CL clients. +func (api *BlockChainAPI) ChainId() *hexutil.Big { + return (*hexutil.Big)(api.b.ChainConfig().ChainID) } // BlockNumber returns the block number of the chain head. -func (s *PublicBlockChainAPI) BlockNumber() hexutil.Uint64 { +func (s *BlockChainAPI) BlockNumber() hexutil.Uint64 { header, _ := s.b.HeaderByNumber(context.Background(), rpc.LatestBlockNumber) // latest header should always be available return hexutil.Uint64(header.Number.Uint64()) } @@ -634,7 +629,7 @@ func (s *PublicBlockChainAPI) BlockNumber() hexutil.Uint64 { // GetBalance returns the amount of wei for the given address in the state of the // given block number. The rpc.LatestBlockNumber and rpc.PendingBlockNumber meta // block numbers are also allowed. -func (s *PublicBlockChainAPI) GetBalance(ctx context.Context, address common.Address, blockNrOrHash rpc.BlockNumberOrHash) (*hexutil.Big, error) { +func (s *BlockChainAPI) GetBalance(ctx context.Context, address common.Address, blockNrOrHash rpc.BlockNumberOrHash) (*hexutil.Big, error) { state, _, err := s.b.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash) if state == nil || err != nil { return nil, err @@ -660,7 +655,7 @@ type StorageResult struct { } // GetProof returns the Merkle-proof for a given account and optionally some storage keys. -func (s *PublicBlockChainAPI) GetProof(ctx context.Context, address common.Address, storageKeys []string, blockNrOrHash rpc.BlockNumberOrHash) (*AccountResult, error) { +func (s *BlockChainAPI) GetProof(ctx context.Context, address common.Address, storageKeys []string, blockNrOrHash rpc.BlockNumberOrHash) (*AccountResult, error) { state, _, err := s.b.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash) if state == nil || err != nil { return nil, err @@ -680,15 +675,19 @@ func (s *PublicBlockChainAPI) GetProof(ctx context.Context, address common.Addre } // create the proof for the storageKeys - for i, key := range storageKeys { + for i, hexKey := range storageKeys { + key, err := decodeHash(hexKey) + if err != nil { + return nil, err + } if storageTrie != nil { - proof, storageError := state.GetStorageProof(address, common.HexToHash(key)) + proof, storageError := state.GetStorageProof(address, key) if storageError != nil { return nil, storageError } - storageProof[i] = StorageResult{key, (*hexutil.Big)(state.GetState(address, common.HexToHash(key)).Big()), toHexSlice(proof)} + storageProof[i] = StorageResult{hexKey, (*hexutil.Big)(state.GetState(address, key).Big()), toHexSlice(proof)} } else { - storageProof[i] = StorageResult{key, &hexutil.Big{}, []string{}} + storageProof[i] = StorageResult{hexKey, &hexutil.Big{}, []string{}} } } @@ -709,10 +708,29 @@ func (s *PublicBlockChainAPI) GetProof(ctx context.Context, address common.Addre }, state.Error() } +// decodeHash parses a hex-encoded 32-byte hash. The input may optionally +// be prefixed by 0x and can have an byte length up to 32. +func decodeHash(s string) (common.Hash, error) { + if strings.HasPrefix(s, "0x") || strings.HasPrefix(s, "0X") { + s = s[2:] + } + if (len(s) & 1) > 0 { + s = "0" + s + } + b, err := hex.DecodeString(s) + if err != nil { + return common.Hash{}, fmt.Errorf("hex string invalid") + } + if len(b) > 32 { + return common.Hash{}, fmt.Errorf("hex string too long, want at most 32 bytes") + } + return common.BytesToHash(b), nil +} + // GetHeaderByNumber returns the requested canonical block header. // * When blockNr is -1 the chain head is returned. // * When blockNr is -2 the pending chain head is returned. -func (s *PublicBlockChainAPI) GetHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (map[string]interface{}, error) { +func (s *BlockChainAPI) GetHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (map[string]interface{}, error) { header, err := s.b.HeaderByNumber(ctx, number) if header != nil && err == nil { response := s.rpcMarshalHeader(ctx, header) @@ -728,7 +746,7 @@ func (s *PublicBlockChainAPI) GetHeaderByNumber(ctx context.Context, number rpc. } // GetHeaderByHash returns the requested header by hash. -func (s *PublicBlockChainAPI) GetHeaderByHash(ctx context.Context, hash common.Hash) map[string]interface{} { +func (s *BlockChainAPI) GetHeaderByHash(ctx context.Context, hash common.Hash) map[string]interface{} { header, _ := s.b.HeaderByHash(ctx, hash) if header != nil { return s.rpcMarshalHeader(ctx, header) @@ -737,11 +755,11 @@ func (s *PublicBlockChainAPI) GetHeaderByHash(ctx context.Context, hash common.H } // GetBlockByNumber returns the requested canonical block. -// * When blockNr is -1 the chain head is returned. -// * When blockNr is -2 the pending chain head is returned. -// * When fullTx is true all transactions in the block are returned, otherwise -// only the transaction hash is returned. -func (s *PublicBlockChainAPI) GetBlockByNumber(ctx context.Context, number rpc.BlockNumber, fullTx bool) (map[string]interface{}, error) { +// - When blockNr is -1 the chain head is returned. +// - When blockNr is -2 the pending chain head is returned. +// - When fullTx is true all transactions in the block are returned, otherwise +// only the transaction hash is returned. +func (s *BlockChainAPI) GetBlockByNumber(ctx context.Context, number rpc.BlockNumber, fullTx bool) (map[string]interface{}, error) { block, err := s.b.BlockByNumber(ctx, number) if block != nil && err == nil { response, err := s.rpcMarshalBlock(ctx, block, true, fullTx) @@ -758,7 +776,7 @@ func (s *PublicBlockChainAPI) GetBlockByNumber(ctx context.Context, number rpc.B // GetBlockByHash returns the requested block. When fullTx is true all transactions in the block are returned in full // detail, otherwise only the transaction hash is returned. -func (s *PublicBlockChainAPI) GetBlockByHash(ctx context.Context, hash common.Hash, fullTx bool) (map[string]interface{}, error) { +func (s *BlockChainAPI) GetBlockByHash(ctx context.Context, hash common.Hash, fullTx bool) (map[string]interface{}, error) { block, err := s.b.BlockByHash(ctx, hash) if block != nil { return s.rpcMarshalBlock(ctx, block, true, fullTx) @@ -767,7 +785,7 @@ func (s *PublicBlockChainAPI) GetBlockByHash(ctx context.Context, hash common.Ha } // GetUncleByBlockNumberAndIndex returns the uncle block for the given block hash and index. -func (s *PublicBlockChainAPI) GetUncleByBlockNumberAndIndex(ctx context.Context, blockNr rpc.BlockNumber, index hexutil.Uint) (map[string]interface{}, error) { +func (s *BlockChainAPI) GetUncleByBlockNumberAndIndex(ctx context.Context, blockNr rpc.BlockNumber, index hexutil.Uint) (map[string]interface{}, error) { block, err := s.b.BlockByNumber(ctx, blockNr) if block != nil { uncles := block.Uncles() @@ -782,7 +800,7 @@ func (s *PublicBlockChainAPI) GetUncleByBlockNumberAndIndex(ctx context.Context, } // GetUncleByBlockHashAndIndex returns the uncle block for the given block hash and index. -func (s *PublicBlockChainAPI) GetUncleByBlockHashAndIndex(ctx context.Context, blockHash common.Hash, index hexutil.Uint) (map[string]interface{}, error) { +func (s *BlockChainAPI) GetUncleByBlockHashAndIndex(ctx context.Context, blockHash common.Hash, index hexutil.Uint) (map[string]interface{}, error) { block, err := s.b.BlockByHash(ctx, blockHash) if block != nil { uncles := block.Uncles() @@ -797,7 +815,7 @@ func (s *PublicBlockChainAPI) GetUncleByBlockHashAndIndex(ctx context.Context, b } // GetUncleCountByBlockNumber returns number of uncles in the block for the given block number -func (s *PublicBlockChainAPI) GetUncleCountByBlockNumber(ctx context.Context, blockNr rpc.BlockNumber) *hexutil.Uint { +func (s *BlockChainAPI) GetUncleCountByBlockNumber(ctx context.Context, blockNr rpc.BlockNumber) *hexutil.Uint { if block, _ := s.b.BlockByNumber(ctx, blockNr); block != nil { n := hexutil.Uint(len(block.Uncles())) return &n @@ -806,7 +824,7 @@ func (s *PublicBlockChainAPI) GetUncleCountByBlockNumber(ctx context.Context, bl } // GetUncleCountByBlockHash returns number of uncles in the block for the given block hash -func (s *PublicBlockChainAPI) GetUncleCountByBlockHash(ctx context.Context, blockHash common.Hash) *hexutil.Uint { +func (s *BlockChainAPI) GetUncleCountByBlockHash(ctx context.Context, blockHash common.Hash) *hexutil.Uint { if block, _ := s.b.BlockByHash(ctx, blockHash); block != nil { n := hexutil.Uint(len(block.Uncles())) return &n @@ -815,7 +833,7 @@ func (s *PublicBlockChainAPI) GetUncleCountByBlockHash(ctx context.Context, bloc } // GetCode returns the code stored at the given address in the state for the given block number. -func (s *PublicBlockChainAPI) GetCode(ctx context.Context, address common.Address, blockNrOrHash rpc.BlockNumberOrHash) (hexutil.Bytes, error) { +func (s *BlockChainAPI) GetCode(ctx context.Context, address common.Address, blockNrOrHash rpc.BlockNumberOrHash) (hexutil.Bytes, error) { state, _, err := s.b.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash) if state == nil || err != nil { return nil, err @@ -827,12 +845,16 @@ func (s *PublicBlockChainAPI) GetCode(ctx context.Context, address common.Addres // GetStorageAt returns the storage from the state at the given address, key and // block number. The rpc.LatestBlockNumber and rpc.PendingBlockNumber meta block // numbers are also allowed. -func (s *PublicBlockChainAPI) GetStorageAt(ctx context.Context, address common.Address, key string, blockNrOrHash rpc.BlockNumberOrHash) (hexutil.Bytes, error) { +func (s *BlockChainAPI) GetStorageAt(ctx context.Context, address common.Address, hexKey string, blockNrOrHash rpc.BlockNumberOrHash) (hexutil.Bytes, error) { state, _, err := s.b.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash) if state == nil || err != nil { return nil, err } - res := state.GetState(address, common.HexToHash(key)) + key, err := decodeHash(hexKey) + if err != nil { + return nil, fmt.Errorf("unable to decode storage key: %s", err) + } + res := state.GetState(address, key) return res[:], state.Error() } @@ -896,6 +918,7 @@ type BlockOverrides struct { GasLimit *hexutil.Uint64 Coinbase *common.Address Random *common.Hash + BaseFee *hexutil.Big } // Apply overrides the given header fields into the given block context. @@ -921,6 +944,9 @@ func (diff *BlockOverrides) Apply(blockCtx *vm.BlockContext) { if diff.Random != nil { blockCtx.Random = diff.Random } + if diff.BaseFee != nil { + blockCtx.BaseFee = diff.BaseFee.ToInt() + } } func DoCall(ctx context.Context, b Backend, args TransactionArgs, blockNrOrHash rpc.BlockNumberOrHash, overrides *StateOverride, timeout time.Duration, globalGasCap uint64) (*core.ExecutionResult, error) { @@ -990,7 +1016,7 @@ func newRevertError(result *core.ExecutionResult) *revertError { } } -// revertError is an API error that encompassas an EVM revertal with JSON error +// revertError is an API error that encompasses an EVM revertal with JSON error // code and a binary data blob. type revertError struct { error @@ -1014,7 +1040,7 @@ func (e *revertError) ErrorData() interface{} { // // Note, this function doesn't make and changes in the state/blockchain and is // useful to execute and retrieve values. -func (s *PublicBlockChainAPI) Call(ctx context.Context, args TransactionArgs, blockNrOrHash rpc.BlockNumberOrHash, overrides *StateOverride) (hexutil.Bytes, error) { +func (s *BlockChainAPI) Call(ctx context.Context, args TransactionArgs, blockNrOrHash rpc.BlockNumberOrHash, overrides *StateOverride) (hexutil.Bytes, error) { result, err := DoCall(ctx, s.b, args, blockNrOrHash, overrides, s.b.RPCEVMTimeout(), s.b.RPCGasCap()) if err != nil { return nil, err @@ -1148,7 +1174,7 @@ func DoEstimateGas(ctx context.Context, b Backend, args TransactionArgs, blockNr // EstimateGas returns an estimate of the amount of gas needed to execute the // given transaction against the current pending block. -func (s *PublicBlockChainAPI) EstimateGas(ctx context.Context, args TransactionArgs, blockNrOrHash *rpc.BlockNumberOrHash) (hexutil.Uint64, error) { +func (s *BlockChainAPI) EstimateGas(ctx context.Context, args TransactionArgs, blockNrOrHash *rpc.BlockNumberOrHash) (hexutil.Uint64, error) { bNrOrHash := rpc.BlockNumberOrHashWithNumber(rpc.PendingBlockNumber) if blockNrOrHash != nil { bNrOrHash = *blockNrOrHash @@ -1222,16 +1248,16 @@ func RPCMarshalBlock(block *types.Block, inclTx bool, fullTx bool, config *param } // rpcMarshalHeader uses the generalized output filler, then adds the total difficulty field, which requires -// a `PublicBlockchainAPI`. -func (s *PublicBlockChainAPI) rpcMarshalHeader(ctx context.Context, header *types.Header) map[string]interface{} { +// a `BlockchainAPI`. +func (s *BlockChainAPI) rpcMarshalHeader(ctx context.Context, header *types.Header) map[string]interface{} { fields := RPCMarshalHeader(header) fields["totalDifficulty"] = (*hexutil.Big)(s.b.GetTd(ctx, header.Hash())) return fields } // rpcMarshalBlock uses the generalized output filler, then adds the total difficulty field, which requires -// a `PublicBlockchainAPI`. -func (s *PublicBlockChainAPI) rpcMarshalBlock(ctx context.Context, b *types.Block, inclTx bool, fullTx bool) (map[string]interface{}, error) { +// a `BlockchainAPI`. +func (s *BlockChainAPI) rpcMarshalBlock(ctx context.Context, b *types.Block, inclTx bool, fullTx bool) (map[string]interface{}, error) { fields, err := RPCMarshalBlock(b, inclTx, fullTx, s.b.ChainConfig()) if err != nil { return nil, err @@ -1268,7 +1294,7 @@ type RPCTransaction struct { // newRPCTransaction returns a transaction that will serialize to the RPC // representation, with the given location metadata set (if available). func newRPCTransaction(tx *types.Transaction, blockHash common.Hash, blockNumber uint64, index uint64, baseFee *big.Int, config *params.ChainConfig) *RPCTransaction { - signer := types.MakeSigner(config, big.NewInt(0).SetUint64(blockNumber)) + signer := types.MakeSigner(config, new(big.Int).SetUint64(blockNumber)) from, _ := types.Sender(signer, tx) v, r, s := tx.RawSignatureValues() result := &RPCTransaction{ @@ -1291,6 +1317,11 @@ func newRPCTransaction(tx *types.Transaction, blockHash common.Hash, blockNumber result.TransactionIndex = (*hexutil.Uint64)(&index) } switch tx.Type() { + case types.LegacyTxType: + // if a legacy transaction has an EIP-155 chain id, include it explicitly + if id := tx.ChainId(); id.Sign() != 0 { + result.ChainID = (*hexutil.Big)(id) + } case types.AccessListTxType: al := tx.AccessList() result.Accesses = &al @@ -1364,7 +1395,7 @@ type accessListResult struct { // CreateAccessList creates a EIP-2930 type AccessList for the given transaction. // Reexec and BlockNrOrHash can be specified to create the accessList on top of a certain state. -func (s *PublicBlockChainAPI) CreateAccessList(ctx context.Context, args TransactionArgs, blockNrOrHash *rpc.BlockNumberOrHash) (*accessListResult, error) { +func (s *BlockChainAPI) CreateAccessList(ctx context.Context, args TransactionArgs, blockNrOrHash *rpc.BlockNumberOrHash) (*accessListResult, error) { bNrOrHash := rpc.BlockNumberOrHashWithNumber(rpc.PendingBlockNumber) if blockNrOrHash != nil { bNrOrHash = *blockNrOrHash @@ -1389,9 +1420,11 @@ func AccessList(ctx context.Context, b Backend, blockNrOrHash rpc.BlockNumberOrH if db == nil || err != nil { return nil, 0, nil, err } - // If the gas amount is not set, extract this as it will depend on access - // lists and we'll need to reestimate every time - nogas := args.Gas == nil + // If the gas amount is not set, default to RPC gas cap. + if args.Gas == nil { + tmp := hexutil.Uint64(b.RPCGasCap()) + args.Gas = &tmp + } // Ensure any missing fields are filled, extract the recipient and input data if err := args.setDefaults(ctx, b); err != nil { @@ -1417,15 +1450,6 @@ func AccessList(ctx context.Context, b Backend, blockNrOrHash rpc.BlockNumberOrH accessList := prevTracer.AccessList() log.Trace("Creating access list", "input", accessList) - // If no gas amount was specified, each unique access list needs it's own - // gas calculation. This is quite expensive, but we need to be accurate - // and it's convered by the sender only anyway. - if nogas { - args.Gas = nil - if err := args.setDefaults(ctx, b); err != nil { - return nil, 0, nil, err // shouldn't happen, just in case - } - } // Copy the original db so we don't modify it statedb := db.Copy() // Set the accesslist to the last al @@ -1453,23 +1477,23 @@ func AccessList(ctx context.Context, b Backend, blockNrOrHash rpc.BlockNumberOrH } } -// PublicTransactionPoolAPI exposes methods for the RPC interface -type PublicTransactionPoolAPI struct { +// TransactionAPI exposes methods for reading and creating transaction data. +type TransactionAPI struct { b Backend nonceLock *AddrLocker signer types.Signer } -// NewPublicTransactionPoolAPI creates a new RPC service with methods specific for the transaction pool. -func NewPublicTransactionPoolAPI(b Backend, nonceLock *AddrLocker) *PublicTransactionPoolAPI { +// NewTransactionAPI creates a new RPC service with methods for interacting with transactions. +func NewTransactionAPI(b Backend, nonceLock *AddrLocker) *TransactionAPI { // The signer used by the API should always be the 'latest' known one because we expect // signers to be backwards-compatible with old transactions. signer := types.LatestSigner(b.ChainConfig()) - return &PublicTransactionPoolAPI{b, nonceLock, signer} + return &TransactionAPI{b, nonceLock, signer} } // GetBlockTransactionCountByNumber returns the number of transactions in the block with the given block number. -func (s *PublicTransactionPoolAPI) GetBlockTransactionCountByNumber(ctx context.Context, blockNr rpc.BlockNumber) *hexutil.Uint { +func (s *TransactionAPI) GetBlockTransactionCountByNumber(ctx context.Context, blockNr rpc.BlockNumber) *hexutil.Uint { if block, _ := s.b.BlockByNumber(ctx, blockNr); block != nil { n := hexutil.Uint(len(block.Transactions())) return &n @@ -1478,7 +1502,7 @@ func (s *PublicTransactionPoolAPI) GetBlockTransactionCountByNumber(ctx context. } // GetBlockTransactionCountByHash returns the number of transactions in the block with the given hash. -func (s *PublicTransactionPoolAPI) GetBlockTransactionCountByHash(ctx context.Context, blockHash common.Hash) *hexutil.Uint { +func (s *TransactionAPI) GetBlockTransactionCountByHash(ctx context.Context, blockHash common.Hash) *hexutil.Uint { if block, _ := s.b.BlockByHash(ctx, blockHash); block != nil { n := hexutil.Uint(len(block.Transactions())) return &n @@ -1487,7 +1511,7 @@ func (s *PublicTransactionPoolAPI) GetBlockTransactionCountByHash(ctx context.Co } // GetTransactionByBlockNumberAndIndex returns the transaction for the given block number and index. -func (s *PublicTransactionPoolAPI) GetTransactionByBlockNumberAndIndex(ctx context.Context, blockNr rpc.BlockNumber, index hexutil.Uint) *RPCTransaction { +func (s *TransactionAPI) GetTransactionByBlockNumberAndIndex(ctx context.Context, blockNr rpc.BlockNumber, index hexutil.Uint) *RPCTransaction { if block, _ := s.b.BlockByNumber(ctx, blockNr); block != nil { return newRPCTransactionFromBlockIndex(block, uint64(index), s.b.ChainConfig()) } @@ -1495,7 +1519,7 @@ func (s *PublicTransactionPoolAPI) GetTransactionByBlockNumberAndIndex(ctx conte } // GetTransactionByBlockHashAndIndex returns the transaction for the given block hash and index. -func (s *PublicTransactionPoolAPI) GetTransactionByBlockHashAndIndex(ctx context.Context, blockHash common.Hash, index hexutil.Uint) *RPCTransaction { +func (s *TransactionAPI) GetTransactionByBlockHashAndIndex(ctx context.Context, blockHash common.Hash, index hexutil.Uint) *RPCTransaction { if block, _ := s.b.BlockByHash(ctx, blockHash); block != nil { return newRPCTransactionFromBlockIndex(block, uint64(index), s.b.ChainConfig()) } @@ -1503,7 +1527,7 @@ func (s *PublicTransactionPoolAPI) GetTransactionByBlockHashAndIndex(ctx context } // GetRawTransactionByBlockNumberAndIndex returns the bytes of the transaction for the given block number and index. -func (s *PublicTransactionPoolAPI) GetRawTransactionByBlockNumberAndIndex(ctx context.Context, blockNr rpc.BlockNumber, index hexutil.Uint) hexutil.Bytes { +func (s *TransactionAPI) GetRawTransactionByBlockNumberAndIndex(ctx context.Context, blockNr rpc.BlockNumber, index hexutil.Uint) hexutil.Bytes { if block, _ := s.b.BlockByNumber(ctx, blockNr); block != nil { return newRPCRawTransactionFromBlockIndex(block, uint64(index)) } @@ -1511,7 +1535,7 @@ func (s *PublicTransactionPoolAPI) GetRawTransactionByBlockNumberAndIndex(ctx co } // GetRawTransactionByBlockHashAndIndex returns the bytes of the transaction for the given block hash and index. -func (s *PublicTransactionPoolAPI) GetRawTransactionByBlockHashAndIndex(ctx context.Context, blockHash common.Hash, index hexutil.Uint) hexutil.Bytes { +func (s *TransactionAPI) GetRawTransactionByBlockHashAndIndex(ctx context.Context, blockHash common.Hash, index hexutil.Uint) hexutil.Bytes { if block, _ := s.b.BlockByHash(ctx, blockHash); block != nil { return newRPCRawTransactionFromBlockIndex(block, uint64(index)) } @@ -1519,7 +1543,7 @@ func (s *PublicTransactionPoolAPI) GetRawTransactionByBlockHashAndIndex(ctx cont } // GetTransactionCount returns the number of transactions the given address has sent for the given block number -func (s *PublicTransactionPoolAPI) GetTransactionCount(ctx context.Context, address common.Address, blockNrOrHash rpc.BlockNumberOrHash) (*hexutil.Uint64, error) { +func (s *TransactionAPI) GetTransactionCount(ctx context.Context, address common.Address, blockNrOrHash rpc.BlockNumberOrHash) (*hexutil.Uint64, error) { // Ask transaction pool for the nonce which includes pending transactions if blockNr, ok := blockNrOrHash.Number(); ok && blockNr == rpc.PendingBlockNumber { nonce, err := s.b.GetPoolNonce(ctx, address) @@ -1538,7 +1562,7 @@ func (s *PublicTransactionPoolAPI) GetTransactionCount(ctx context.Context, addr } // GetTransactionByHash returns the transaction for the given hash -func (s *PublicTransactionPoolAPI) GetTransactionByHash(ctx context.Context, hash common.Hash) (*RPCTransaction, error) { +func (s *TransactionAPI) GetTransactionByHash(ctx context.Context, hash common.Hash) (*RPCTransaction, error) { // Try to return an already finalized transaction tx, blockHash, blockNumber, index, err := s.b.GetTransaction(ctx, hash) if err != nil { @@ -1561,7 +1585,7 @@ func (s *PublicTransactionPoolAPI) GetTransactionByHash(ctx context.Context, has } // GetRawTransactionByHash returns the bytes of the transaction for the given hash. -func (s *PublicTransactionPoolAPI) GetRawTransactionByHash(ctx context.Context, hash common.Hash) (hexutil.Bytes, error) { +func (s *TransactionAPI) GetRawTransactionByHash(ctx context.Context, hash common.Hash) (hexutil.Bytes, error) { // Retrieve a finalized transaction, or a pooled otherwise tx, _, _, _, err := s.b.GetTransaction(ctx, hash) if err != nil { @@ -1578,9 +1602,11 @@ func (s *PublicTransactionPoolAPI) GetRawTransactionByHash(ctx context.Context, } // GetTransactionReceipt returns the transaction receipt for the given transaction hash. -func (s *PublicTransactionPoolAPI) GetTransactionReceipt(ctx context.Context, hash common.Hash) (map[string]interface{}, error) { +func (s *TransactionAPI) GetTransactionReceipt(ctx context.Context, hash common.Hash) (map[string]interface{}, error) { tx, blockHash, blockNumber, index, err := s.b.GetTransaction(ctx, hash) if err != nil { + // When the transaction doesn't exist, the RPC method should return JSON null + // as per specification. return nil, nil } receipts, err := s.b.GetReceipts(ctx, blockHash) @@ -1639,7 +1665,7 @@ func (s *PublicTransactionPoolAPI) GetTransactionReceipt(ctx context.Context, ha } // sign is a helper function that signs a transaction with the private key of the given address. -func (s *PublicTransactionPoolAPI) sign(addr common.Address, tx *types.Transaction) (*types.Transaction, error) { +func (s *TransactionAPI) sign(addr common.Address, tx *types.Transaction) (*types.Transaction, error) { // Look up the wallet containing the requested signer account := accounts.Account{Address: addr} @@ -1683,7 +1709,7 @@ func SubmitTransaction(ctx context.Context, b Backend, tx *types.Transaction) (c // SendTransaction creates a transaction for the given argument, sign it and submit it to the // transaction pool. -func (s *PublicTransactionPoolAPI) SendTransaction(ctx context.Context, args TransactionArgs) (common.Hash, error) { +func (s *TransactionAPI) SendTransaction(ctx context.Context, args TransactionArgs) (common.Hash, error) { // Look up the wallet containing the requested signer account := accounts.Account{Address: args.from()} @@ -1716,7 +1742,7 @@ func (s *PublicTransactionPoolAPI) SendTransaction(ctx context.Context, args Tra // FillTransaction fills the defaults (nonce, gas, gasPrice or 1559 fields) // on a given unsigned transaction, and returns it to the caller for further // processing (signing + broadcast). -func (s *PublicTransactionPoolAPI) FillTransaction(ctx context.Context, args TransactionArgs) (*SignTransactionResult, error) { +func (s *TransactionAPI) FillTransaction(ctx context.Context, args TransactionArgs) (*SignTransactionResult, error) { // Set some sanity defaults and terminate on failure if err := args.setDefaults(ctx, s.b); err != nil { return nil, err @@ -1732,7 +1758,7 @@ func (s *PublicTransactionPoolAPI) FillTransaction(ctx context.Context, args Tra // SendRawTransaction will add the signed transaction to the transaction pool. // The sender is responsible for signing the transaction and using the correct nonce. -func (s *PublicTransactionPoolAPI) SendRawTransaction(ctx context.Context, input hexutil.Bytes) (common.Hash, error) { +func (s *TransactionAPI) SendRawTransaction(ctx context.Context, input hexutil.Bytes) (common.Hash, error) { tx := new(types.Transaction) if err := tx.UnmarshalBinary(input); err != nil { return common.Hash{}, err @@ -1741,7 +1767,7 @@ func (s *PublicTransactionPoolAPI) SendRawTransaction(ctx context.Context, input } // Sign calculates an ECDSA signature for: -// keccack256("\x19Ethereum Signed Message:\n" + len(message) + message). +// keccak256("\x19Ethereum Signed Message:\n" + len(message) + message). // // Note, the produced signature conforms to the secp256k1 curve R, S and V values, // where the V value will be 27 or 28 for legacy reasons. @@ -1749,7 +1775,7 @@ func (s *PublicTransactionPoolAPI) SendRawTransaction(ctx context.Context, input // The account associated with addr must be unlocked. // // https://github.com/ethereum/wiki/wiki/JSON-RPC#eth_sign -func (s *PublicTransactionPoolAPI) Sign(addr common.Address, data hexutil.Bytes) (hexutil.Bytes, error) { +func (s *TransactionAPI) Sign(addr common.Address, data hexutil.Bytes) (hexutil.Bytes, error) { // Look up the wallet containing the requested signer account := accounts.Account{Address: addr} @@ -1774,7 +1800,7 @@ type SignTransactionResult struct { // SignTransaction will sign the given transaction with the from account. // The node needs to have the private key of the account corresponding with // the given from address and it needs to be unlocked. -func (s *PublicTransactionPoolAPI) SignTransaction(ctx context.Context, args TransactionArgs) (*SignTransactionResult, error) { +func (s *TransactionAPI) SignTransaction(ctx context.Context, args TransactionArgs) (*SignTransactionResult, error) { if args.Gas == nil { return nil, fmt.Errorf("gas not specified") } @@ -1805,7 +1831,7 @@ func (s *PublicTransactionPoolAPI) SignTransaction(ctx context.Context, args Tra // PendingTransactions returns the transactions that are in the transaction pool // and have a from address that is one of the accounts this node manages. -func (s *PublicTransactionPoolAPI) PendingTransactions() ([]*RPCTransaction, error) { +func (s *TransactionAPI) PendingTransactions() ([]*RPCTransaction, error) { pending, err := s.b.GetPoolTransactions() if err != nil { return nil, err @@ -1829,7 +1855,7 @@ func (s *PublicTransactionPoolAPI) PendingTransactions() ([]*RPCTransaction, err // Resend accepts an existing transaction and a new gas price and limit. It will remove // the given transaction from the pool and reinsert it with the new gas price and limit. -func (s *PublicTransactionPoolAPI) Resend(ctx context.Context, sendArgs TransactionArgs, gasPrice *hexutil.Big, gasLimit *hexutil.Uint64) (common.Hash, error) { +func (s *TransactionAPI) Resend(ctx context.Context, sendArgs TransactionArgs, gasPrice *hexutil.Big, gasLimit *hexutil.Uint64) (common.Hash, error) { if sendArgs.Nonce == nil { return common.Hash{}, fmt.Errorf("missing transaction nonce in transaction spec") } @@ -1879,38 +1905,57 @@ func (s *PublicTransactionPoolAPI) Resend(ctx context.Context, sendArgs Transact return common.Hash{}, fmt.Errorf("transaction %#x not found", matchTx.Hash()) } -// PublicDebugAPI is the collection of Ethereum APIs exposed over the public -// debugging endpoint. -type PublicDebugAPI struct { +// DebugAPI is the collection of Ethereum APIs exposed over the debugging +// namespace. +type DebugAPI struct { b Backend } -// NewPublicDebugAPI creates a new API definition for the public debug methods -// of the Ethereum service. -func NewPublicDebugAPI(b Backend) *PublicDebugAPI { - return &PublicDebugAPI{b: b} +// NewDebugAPI creates a new instance of DebugAPI. +func NewDebugAPI(b Backend) *DebugAPI { + return &DebugAPI{b: b} } -// GetHeaderRlp retrieves the RLP encoded for of a single header. -func (api *PublicDebugAPI) GetHeaderRlp(ctx context.Context, number uint64) (hexutil.Bytes, error) { - header, _ := api.b.HeaderByNumber(ctx, rpc.BlockNumber(number)) +// GetRawHeader retrieves the RLP encoding for a single header. +func (api *DebugAPI) GetRawHeader(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (hexutil.Bytes, error) { + var hash common.Hash + if h, ok := blockNrOrHash.Hash(); ok { + hash = h + } else { + block, err := api.b.BlockByNumberOrHash(ctx, blockNrOrHash) + if err != nil { + return nil, err + } + hash = block.Hash() + } + header, _ := api.b.HeaderByHash(ctx, hash) if header == nil { - return nil, fmt.Errorf("header #%d not found", number) + return nil, fmt.Errorf("header #%d not found", hash) } return rlp.EncodeToBytes(header) } -// GetBlockRlp retrieves the RLP encoded for of a single block. -func (api *PublicDebugAPI) GetBlockRlp(ctx context.Context, number uint64) (hexutil.Bytes, error) { - block, _ := api.b.BlockByNumber(ctx, rpc.BlockNumber(number)) +// GetRawBlock retrieves the RLP encoded for a single block. +func (api *DebugAPI) GetRawBlock(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (hexutil.Bytes, error) { + var hash common.Hash + if h, ok := blockNrOrHash.Hash(); ok { + hash = h + } else { + block, err := api.b.BlockByNumberOrHash(ctx, blockNrOrHash) + if err != nil { + return nil, err + } + hash = block.Hash() + } + block, _ := api.b.BlockByHash(ctx, hash) if block == nil { - return nil, fmt.Errorf("block #%d not found", number) + return nil, fmt.Errorf("block #%d not found", hash) } return rlp.EncodeToBytes(block) } -// GetRawReceipts retrieves the binary-encoded raw receipts of a single block. -func (api *PublicDebugAPI) GetRawReceipts(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) ([]hexutil.Bytes, error) { +// GetRawReceipts retrieves the binary-encoded receipts of a single block. +func (api *DebugAPI) GetRawReceipts(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) ([]hexutil.Bytes, error) { var hash common.Hash if h, ok := blockNrOrHash.Hash(); ok { hash = h @@ -1936,8 +1981,24 @@ func (api *PublicDebugAPI) GetRawReceipts(ctx context.Context, blockNrOrHash rpc return result, nil } +// GetRawTransaction returns the bytes of the transaction for the given hash. +func (s *DebugAPI) GetRawTransaction(ctx context.Context, hash common.Hash) (hexutil.Bytes, error) { + // Retrieve a finalized transaction, or a pooled otherwise + tx, _, _, _, err := s.b.GetTransaction(ctx, hash) + if err != nil { + return nil, err + } + if tx == nil { + if tx = s.b.GetPoolTransaction(hash); tx == nil { + // Transaction not found anywhere, abort + return nil, nil + } + } + return tx.MarshalBinary() +} + // PrintBlock retrieves a block and returns its pretty printed form. -func (api *PublicDebugAPI) PrintBlock(ctx context.Context, number uint64) (string, error) { +func (api *DebugAPI) PrintBlock(ctx context.Context, number uint64) (string, error) { block, _ := api.b.BlockByNumber(ctx, rpc.BlockNumber(number)) if block == nil { return "", fmt.Errorf("block #%d not found", number) @@ -1946,28 +2007,16 @@ func (api *PublicDebugAPI) PrintBlock(ctx context.Context, number uint64) (strin } // SeedHash retrieves the seed hash of a block. -func (api *PublicDebugAPI) SeedHash(ctx context.Context, number uint64) (string, error) { +func (api *DebugAPI) SeedHash(ctx context.Context, number uint64) (string, error) { block, _ := api.b.BlockByNumber(ctx, rpc.BlockNumber(number)) if block == nil { return "", fmt.Errorf("block #%d not found", number) } - return fmt.Sprintf("0x%x", ethash.SeedHash(number)), nil -} - -// PrivateDebugAPI is the collection of Ethereum APIs exposed over the private -// debugging endpoint. -type PrivateDebugAPI struct { - b Backend -} - -// NewPrivateDebugAPI creates a new API definition for the private debug methods -// of the Ethereum service. -func NewPrivateDebugAPI(b Backend) *PrivateDebugAPI { - return &PrivateDebugAPI{b: b} + return fmt.Sprintf("%#x", ethash.SeedHash(number)), nil } // ChaindbProperty returns leveldb properties of the key-value database. -func (api *PrivateDebugAPI) ChaindbProperty(property string) (string, error) { +func (api *DebugAPI) ChaindbProperty(property string) (string, error) { if property == "" { property = "leveldb.stats" } else if !strings.HasPrefix(property, "leveldb.") { @@ -1978,7 +2027,7 @@ func (api *PrivateDebugAPI) ChaindbProperty(property string) (string, error) { // ChaindbCompact flattens the entire key-value database into a single level, // removing all unused slots and merging all keys. -func (api *PrivateDebugAPI) ChaindbCompact() error { +func (api *DebugAPI) ChaindbCompact() error { for b := byte(0); b < 255; b++ { log.Info("Compacting chain database", "range", fmt.Sprintf("0x%0.2X-0x%0.2X", b, b+1)) if err := api.b.ChainDb().Compact([]byte{b}, []byte{b + 1}); err != nil { @@ -1990,33 +2039,33 @@ func (api *PrivateDebugAPI) ChaindbCompact() error { } // SetHead rewinds the head of the blockchain to a previous block. -func (api *PrivateDebugAPI) SetHead(number hexutil.Uint64) { +func (api *DebugAPI) SetHead(number hexutil.Uint64) { api.b.SetHead(uint64(number)) } -// PublicNetAPI offers network related RPC methods -type PublicNetAPI struct { +// NetAPI offers network related RPC methods +type NetAPI struct { net *p2p.Server networkVersion uint64 } -// NewPublicNetAPI creates a new net API instance. -func NewPublicNetAPI(net *p2p.Server, networkVersion uint64) *PublicNetAPI { - return &PublicNetAPI{net, networkVersion} +// NewNetAPI creates a new net API instance. +func NewNetAPI(net *p2p.Server, networkVersion uint64) *NetAPI { + return &NetAPI{net, networkVersion} } // Listening returns an indication if the node is listening for network connections. -func (s *PublicNetAPI) Listening() bool { +func (s *NetAPI) Listening() bool { return true // always listening } // PeerCount returns the number of connected peers -func (s *PublicNetAPI) PeerCount() hexutil.Uint { +func (s *NetAPI) PeerCount() hexutil.Uint { return hexutil.Uint(s.net.PeerCount()) } // Version returns the current ethereum protocol version. -func (s *PublicNetAPI) Version() string { +func (s *NetAPI) Version() string { return fmt.Sprintf("%d", s.networkVersion) } diff --git a/internal/ethapi/backend.go b/internal/ethapi/backend.go index bc60fb2a64f6c..5b4ceb6310691 100644 --- a/internal/ethapi/backend.go +++ b/internal/ethapi/backend.go @@ -27,10 +27,10 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/bloombits" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/params" @@ -65,6 +65,7 @@ type Backend interface { BlockByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Block, error) StateAndHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*state.StateDB, *types.Header, error) StateAndHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*state.StateDB, *types.Header, error) + PendingBlockAndReceipts() (*types.Block, types.Receipts) GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) GetTd(ctx context.Context, hash common.Hash) *big.Int GetEVM(ctx context.Context, msg core.Message, state *state.StateDB, header *types.Header, vmConfig *vm.Config) (*vm.EVM, func() error, error) @@ -83,16 +84,12 @@ type Backend interface { TxPoolContentFrom(addr common.Address) (types.Transactions, types.Transactions) SubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscription - // Filter API - BloomStatus() (uint64, uint64) - GetLogs(ctx context.Context, blockHash common.Hash) ([][]*types.Log, error) - ServiceFilter(ctx context.Context, session *bloombits.MatcherSession) - SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription - SubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription - SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription - ChainConfig() *params.ChainConfig Engine() consensus.Engine + + // eth/filters needs to be initialized from this backend type, so methods needed by + // it must also be included here. + filters.Backend } func GetAPIs(apiBackend Backend) []rpc.API { @@ -100,43 +97,25 @@ func GetAPIs(apiBackend Backend) []rpc.API { return []rpc.API{ { Namespace: "eth", - Version: "1.0", - Service: NewPublicEthereumAPI(apiBackend), - Public: true, + Service: NewEthereumAPI(apiBackend), }, { Namespace: "eth", - Version: "1.0", - Service: NewPublicBlockChainAPI(apiBackend), - Public: true, + Service: NewBlockChainAPI(apiBackend), }, { Namespace: "eth", - Version: "1.0", - Service: NewPublicTransactionPoolAPI(apiBackend, nonceLock), - Public: true, + Service: NewTransactionAPI(apiBackend, nonceLock), }, { Namespace: "txpool", - Version: "1.0", - Service: NewPublicTxPoolAPI(apiBackend), - Public: true, - }, { - Namespace: "debug", - Version: "1.0", - Service: NewPublicDebugAPI(apiBackend), - Public: true, + Service: NewTxPoolAPI(apiBackend), }, { Namespace: "debug", - Version: "1.0", - Service: NewPrivateDebugAPI(apiBackend), + Service: NewDebugAPI(apiBackend), }, { Namespace: "eth", - Version: "1.0", - Service: NewPublicAccountAPI(apiBackend.AccountManager()), - Public: true, + Service: NewEthereumAccountAPI(apiBackend.AccountManager()), }, { Namespace: "personal", - Version: "1.0", - Service: NewPrivateAccountAPI(apiBackend, nonceLock), - Public: false, + Service: NewPersonalAccountAPI(apiBackend, nonceLock), }, } } diff --git a/internal/ethapi/dbapi.go b/internal/ethapi/dbapi.go index 33dca29d3c66f..33fda936dcd0f 100644 --- a/internal/ethapi/dbapi.go +++ b/internal/ethapi/dbapi.go @@ -22,7 +22,7 @@ import ( ) // DbGet returns the raw value of a key stored in the database. -func (api *PrivateDebugAPI) DbGet(key string) (hexutil.Bytes, error) { +func (api *DebugAPI) DbGet(key string) (hexutil.Bytes, error) { blob, err := common.ParseHexOrString(key) if err != nil { return nil, err @@ -32,12 +32,12 @@ func (api *PrivateDebugAPI) DbGet(key string) (hexutil.Bytes, error) { // DbAncient retrieves an ancient binary blob from the append-only immutable files. // It is a mapping to the `AncientReaderOp.Ancient` method -func (api *PrivateDebugAPI) DbAncient(kind string, number uint64) (hexutil.Bytes, error) { +func (api *DebugAPI) DbAncient(kind string, number uint64) (hexutil.Bytes, error) { return api.b.ChainDb().Ancient(kind, number) } // DbAncients returns the ancient item numbers in the ancient store. // It is a mapping to the `AncientReaderOp.Ancients` method -func (api *PrivateDebugAPI) DbAncients() (uint64, error) { +func (api *DebugAPI) DbAncients() (uint64, error) { return api.b.ChainDb().Ancients() } diff --git a/internal/ethapi/transaction_args.go b/internal/ethapi/transaction_args.go index 9c5950af58fe6..e07248db5d693 100644 --- a/internal/ethapi/transaction_args.go +++ b/internal/ethapi/transaction_args.go @@ -75,56 +75,8 @@ func (args *TransactionArgs) data() []byte { // setDefaults fills in default values for unspecified tx fields. func (args *TransactionArgs) setDefaults(ctx context.Context, b Backend) error { - if args.GasPrice != nil && (args.MaxFeePerGas != nil || args.MaxPriorityFeePerGas != nil) { - return errors.New("both gasPrice and (maxFeePerGas or maxPriorityFeePerGas) specified") - } - // After london, default to 1559 unless gasPrice is set - head := b.CurrentHeader() - // If user specifies both maxPriorityfee and maxFee, then we do not - // need to consult the chain for defaults. It's definitely a London tx. - if args.MaxPriorityFeePerGas == nil || args.MaxFeePerGas == nil { - // In this clause, user left some fields unspecified. - if b.ChainConfig().IsLondon(head.Number) && args.GasPrice == nil { - if args.MaxPriorityFeePerGas == nil { - tip, err := b.SuggestGasTipCap(ctx) - if err != nil { - return err - } - args.MaxPriorityFeePerGas = (*hexutil.Big)(tip) - } - if args.MaxFeePerGas == nil { - gasFeeCap := new(big.Int).Add( - (*big.Int)(args.MaxPriorityFeePerGas), - new(big.Int).Mul(head.BaseFee, big.NewInt(2)), - ) - args.MaxFeePerGas = (*hexutil.Big)(gasFeeCap) - } - if args.MaxFeePerGas.ToInt().Cmp(args.MaxPriorityFeePerGas.ToInt()) < 0 { - return fmt.Errorf("maxFeePerGas (%v) < maxPriorityFeePerGas (%v)", args.MaxFeePerGas, args.MaxPriorityFeePerGas) - } - } else { - if args.MaxFeePerGas != nil || args.MaxPriorityFeePerGas != nil { - return errors.New("maxFeePerGas or maxPriorityFeePerGas specified but london is not active yet") - } - if args.GasPrice == nil { - price, err := b.SuggestGasTipCap(ctx) - if err != nil { - return err - } - if b.ChainConfig().IsLondon(head.Number) { - // The legacy tx gas price suggestion should not add 2x base fee - // because all fees are consumed, so it would result in a spiral - // upwards. - price.Add(price, head.BaseFee) - } - args.GasPrice = (*hexutil.Big)(price) - } - } - } else { - // Both maxPriorityfee and maxFee set by caller. Sanity-check their internal relation - if args.MaxFeePerGas.ToInt().Cmp(args.MaxPriorityFeePerGas.ToInt()) < 0 { - return fmt.Errorf("maxFeePerGas (%v) < maxPriorityFeePerGas (%v)", args.MaxFeePerGas, args.MaxPriorityFeePerGas) - } + if err := args.setFeeDefaults(ctx, b); err != nil { + return err } if args.Value == nil { args.Value = new(hexutil.Big) @@ -165,9 +117,81 @@ func (args *TransactionArgs) setDefaults(ctx context.Context, b Backend) error { args.Gas = &estimated log.Trace("Estimate gas usage automatically", "gas", args.Gas) } - if args.ChainID == nil { - id := (*hexutil.Big)(b.ChainConfig().ChainID) - args.ChainID = id + // If chain id is provided, ensure it matches the local chain id. Otherwise, set the local + // chain id as the default. + want := b.ChainConfig().ChainID + if args.ChainID != nil { + if have := (*big.Int)(args.ChainID); have.Cmp(want) != 0 { + return fmt.Errorf("chainId does not match node's (have=%v, want=%v)", have, want) + } + } else { + args.ChainID = (*hexutil.Big)(want) + } + return nil +} + +// setFeeDefaults fills in default fee values for unspecified tx fields. +func (args *TransactionArgs) setFeeDefaults(ctx context.Context, b Backend) error { + // If both gasPrice and at least one of the EIP-1559 fee parameters are specified, error. + if args.GasPrice != nil && (args.MaxFeePerGas != nil || args.MaxPriorityFeePerGas != nil) { + return errors.New("both gasPrice and (maxFeePerGas or maxPriorityFeePerGas) specified") + } + // If the tx has completely specified a fee mechanism, no default is needed. This allows users + // who are not yet synced past London to get defaults for other tx values. See + // https://github.com/ethereum/go-ethereum/pull/23274 for more information. + eip1559ParamsSet := args.MaxFeePerGas != nil && args.MaxPriorityFeePerGas != nil + if (args.GasPrice != nil && !eip1559ParamsSet) || (args.GasPrice == nil && eip1559ParamsSet) { + // Sanity check the EIP-1559 fee parameters if present. + if args.GasPrice == nil && args.MaxFeePerGas.ToInt().Cmp(args.MaxPriorityFeePerGas.ToInt()) < 0 { + return fmt.Errorf("maxFeePerGas (%v) < maxPriorityFeePerGas (%v)", args.MaxFeePerGas, args.MaxPriorityFeePerGas) + } + return nil + } + // Now attempt to fill in default value depending on whether London is active or not. + head := b.CurrentHeader() + if b.ChainConfig().IsLondon(head.Number) { + // London is active, set maxPriorityFeePerGas and maxFeePerGas. + if err := args.setLondonFeeDefaults(ctx, head, b); err != nil { + return err + } + } else { + if args.MaxFeePerGas != nil || args.MaxPriorityFeePerGas != nil { + return fmt.Errorf("maxFeePerGas and maxPriorityFeePerGas are not valid before London is active") + } + // London not active, set gas price. + price, err := b.SuggestGasTipCap(ctx) + if err != nil { + return err + } + args.GasPrice = (*hexutil.Big)(price) + } + return nil +} + +// setLondonFeeDefaults fills in reasonable default fee values for unspecified fields. +func (args *TransactionArgs) setLondonFeeDefaults(ctx context.Context, head *types.Header, b Backend) error { + // Set maxPriorityFeePerGas if it is missing. + if args.MaxPriorityFeePerGas == nil { + tip, err := b.SuggestGasTipCap(ctx) + if err != nil { + return err + } + args.MaxPriorityFeePerGas = (*hexutil.Big)(tip) + } + // Set maxFeePerGas if it is missing. + if args.MaxFeePerGas == nil { + // Set the max fee to be 2 times larger than the previous block's base fee. + // The additional slack allows the tx to not become invalidated if the base + // fee is rising. + val := new(big.Int).Add( + args.MaxPriorityFeePerGas.ToInt(), + new(big.Int).Mul(head.BaseFee, big.NewInt(2)), + ) + args.MaxFeePerGas = (*hexutil.Big)(val) + } + // Both EIP-1559 fee parameters are now set; sanity check them. + if args.MaxFeePerGas.ToInt().Cmp(args.MaxPriorityFeePerGas.ToInt()) < 0 { + return fmt.Errorf("maxFeePerGas (%v) < maxPriorityFeePerGas (%v)", args.MaxFeePerGas, args.MaxPriorityFeePerGas) } return nil } @@ -214,7 +238,7 @@ func (args *TransactionArgs) ToMessage(globalGasCap uint64, baseFee *big.Int) (t gasPrice = args.GasPrice.ToInt() gasFeeCap, gasTipCap = gasPrice, gasPrice } else { - // User specified 1559 gas feilds (or none), use those + // User specified 1559 gas fields (or none), use those gasFeeCap = new(big.Int) if args.MaxFeePerGas != nil { gasFeeCap = args.MaxFeePerGas.ToInt() diff --git a/internal/ethapi/transaction_args_test.go b/internal/ethapi/transaction_args_test.go new file mode 100644 index 0000000000000..28dc561c36e4c --- /dev/null +++ b/internal/ethapi/transaction_args_test.go @@ -0,0 +1,342 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package ethapi + +import ( + "context" + "fmt" + "math/big" + "reflect" + "testing" + "time" + + "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/accounts" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/consensus" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/bloombits" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/event" + "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/rpc" +) + +// TestSetFeeDefaults tests the logic for filling in default fee values works as expected. +func TestSetFeeDefaults(t *testing.T) { + type test struct { + name string + isLondon bool + in *TransactionArgs + want *TransactionArgs + err error + } + + var ( + b = newBackendMock() + fortytwo = (*hexutil.Big)(big.NewInt(42)) + maxFee = (*hexutil.Big)(new(big.Int).Add(new(big.Int).Mul(b.current.BaseFee, big.NewInt(2)), fortytwo.ToInt())) + al = &types.AccessList{types.AccessTuple{Address: common.Address{0xaa}, StorageKeys: []common.Hash{{0x01}}}} + ) + + tests := []test{ + // Legacy txs + { + "legacy tx pre-London", + false, + &TransactionArgs{}, + &TransactionArgs{GasPrice: fortytwo}, + nil, + }, + { + "legacy tx post-London, explicit gas price", + true, + &TransactionArgs{GasPrice: fortytwo}, + &TransactionArgs{GasPrice: fortytwo}, + nil, + }, + + // Access list txs + { + "access list tx pre-London", + false, + &TransactionArgs{AccessList: al}, + &TransactionArgs{AccessList: al, GasPrice: fortytwo}, + nil, + }, + { + "access list tx post-London, explicit gas price", + false, + &TransactionArgs{AccessList: al, GasPrice: fortytwo}, + &TransactionArgs{AccessList: al, GasPrice: fortytwo}, + nil, + }, + { + "access list tx post-London", + true, + &TransactionArgs{AccessList: al}, + &TransactionArgs{AccessList: al, MaxFeePerGas: maxFee, MaxPriorityFeePerGas: fortytwo}, + nil, + }, + { + "access list tx post-London, only max fee", + true, + &TransactionArgs{AccessList: al, MaxFeePerGas: maxFee}, + &TransactionArgs{AccessList: al, MaxFeePerGas: maxFee, MaxPriorityFeePerGas: fortytwo}, + nil, + }, + { + "access list tx post-London, only priority fee", + true, + &TransactionArgs{AccessList: al, MaxFeePerGas: maxFee}, + &TransactionArgs{AccessList: al, MaxFeePerGas: maxFee, MaxPriorityFeePerGas: fortytwo}, + nil, + }, + + // Dynamic fee txs + { + "dynamic tx post-London", + true, + &TransactionArgs{}, + &TransactionArgs{MaxFeePerGas: maxFee, MaxPriorityFeePerGas: fortytwo}, + nil, + }, + { + "dynamic tx post-London, only max fee", + true, + &TransactionArgs{MaxFeePerGas: maxFee}, + &TransactionArgs{MaxFeePerGas: maxFee, MaxPriorityFeePerGas: fortytwo}, + nil, + }, + { + "dynamic tx post-London, only priority fee", + true, + &TransactionArgs{MaxFeePerGas: maxFee}, + &TransactionArgs{MaxFeePerGas: maxFee, MaxPriorityFeePerGas: fortytwo}, + nil, + }, + { + "dynamic fee tx pre-London, maxFee set", + false, + &TransactionArgs{MaxFeePerGas: maxFee}, + nil, + fmt.Errorf("maxFeePerGas and maxPriorityFeePerGas are not valid before London is active"), + }, + { + "dynamic fee tx pre-London, priorityFee set", + false, + &TransactionArgs{MaxPriorityFeePerGas: fortytwo}, + nil, + fmt.Errorf("maxFeePerGas and maxPriorityFeePerGas are not valid before London is active"), + }, + { + "dynamic fee tx, maxFee < priorityFee", + true, + &TransactionArgs{MaxFeePerGas: maxFee, MaxPriorityFeePerGas: (*hexutil.Big)(big.NewInt(1000))}, + nil, + fmt.Errorf("maxFeePerGas (0x3e) < maxPriorityFeePerGas (0x3e8)"), + }, + { + "dynamic fee tx, maxFee < priorityFee while setting default", + true, + &TransactionArgs{MaxFeePerGas: (*hexutil.Big)(big.NewInt(7))}, + nil, + fmt.Errorf("maxFeePerGas (0x7) < maxPriorityFeePerGas (0x2a)"), + }, + + // Misc + { + "set all fee parameters", + false, + &TransactionArgs{GasPrice: fortytwo, MaxFeePerGas: maxFee, MaxPriorityFeePerGas: fortytwo}, + nil, + fmt.Errorf("both gasPrice and (maxFeePerGas or maxPriorityFeePerGas) specified"), + }, + { + "set gas price and maxPriorityFee", + false, + &TransactionArgs{GasPrice: fortytwo, MaxPriorityFeePerGas: fortytwo}, + nil, + fmt.Errorf("both gasPrice and (maxFeePerGas or maxPriorityFeePerGas) specified"), + }, + { + "set gas price and maxFee", + true, + &TransactionArgs{GasPrice: fortytwo, MaxFeePerGas: maxFee}, + nil, + fmt.Errorf("both gasPrice and (maxFeePerGas or maxPriorityFeePerGas) specified"), + }, + } + + ctx := context.Background() + for i, test := range tests { + if test.isLondon { + b.activateLondon() + } else { + b.deactivateLondon() + } + got := test.in + err := got.setFeeDefaults(ctx, b) + if err != nil && err.Error() == test.err.Error() { + // Test threw expected error. + continue + } else if err != nil { + t.Fatalf("test %d (%s): unexpected error: %s", i, test.name, err) + } + if !reflect.DeepEqual(got, test.want) { + t.Fatalf("test %d (%s): did not fill defaults as expected: (got: %v, want: %v)", i, test.name, got, test.want) + } + } +} + +type backendMock struct { + current *types.Header + config *params.ChainConfig +} + +func newBackendMock() *backendMock { + config := ¶ms.ChainConfig{ + ChainID: big.NewInt(42), + HomesteadBlock: big.NewInt(0), + DAOForkBlock: nil, + DAOForkSupport: true, + EIP150Block: big.NewInt(0), + EIP155Block: big.NewInt(0), + EIP158Block: big.NewInt(0), + ByzantiumBlock: big.NewInt(0), + ConstantinopleBlock: big.NewInt(0), + PetersburgBlock: big.NewInt(0), + IstanbulBlock: big.NewInt(0), + MuirGlacierBlock: big.NewInt(0), + BerlinBlock: big.NewInt(0), + LondonBlock: big.NewInt(1000), + } + return &backendMock{ + current: &types.Header{ + Difficulty: big.NewInt(10000000000), + Number: big.NewInt(1100), + GasLimit: 8_000_000, + GasUsed: 8_000_000, + Time: 555, + Extra: make([]byte, 32), + BaseFee: big.NewInt(10), + }, + config: config, + } +} + +func (b *backendMock) activateLondon() { + b.current.Number = big.NewInt(1100) +} + +func (b *backendMock) deactivateLondon() { + b.current.Number = big.NewInt(900) +} +func (b *backendMock) SuggestGasTipCap(ctx context.Context) (*big.Int, error) { + return big.NewInt(42), nil +} +func (b *backendMock) CurrentHeader() *types.Header { return b.current } +func (b *backendMock) ChainConfig() *params.ChainConfig { return b.config } + +// Other methods needed to implement Backend interface. +func (b *backendMock) SyncProgress() ethereum.SyncProgress { return ethereum.SyncProgress{} } +func (b *backendMock) FeeHistory(ctx context.Context, blockCount int, lastBlock rpc.BlockNumber, rewardPercentiles []float64) (*big.Int, [][]*big.Int, []*big.Int, []float64, error) { + return nil, nil, nil, nil, nil +} +func (b *backendMock) ChainDb() ethdb.Database { return nil } +func (b *backendMock) AccountManager() *accounts.Manager { return nil } +func (b *backendMock) ExtRPCEnabled() bool { return false } +func (b *backendMock) RPCGasCap() uint64 { return 0 } +func (b *backendMock) RPCEVMTimeout() time.Duration { return time.Second } +func (b *backendMock) RPCTxFeeCap() float64 { return 0 } +func (b *backendMock) UnprotectedAllowed() bool { return false } +func (b *backendMock) SetHead(number uint64) {} +func (b *backendMock) HeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Header, error) { + return nil, nil +} +func (b *backendMock) HeaderByHash(ctx context.Context, hash common.Hash) (*types.Header, error) { + return nil, nil +} +func (b *backendMock) HeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Header, error) { + return nil, nil +} +func (b *backendMock) CurrentBlock() *types.Block { return nil } +func (b *backendMock) BlockByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Block, error) { + return nil, nil +} +func (b *backendMock) BlockByHash(ctx context.Context, hash common.Hash) (*types.Block, error) { + return nil, nil +} +func (b *backendMock) BlockByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Block, error) { + return nil, nil +} +func (b *backendMock) StateAndHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*state.StateDB, *types.Header, error) { + return nil, nil, nil +} +func (b *backendMock) StateAndHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*state.StateDB, *types.Header, error) { + return nil, nil, nil +} +func (b *backendMock) PendingBlockAndReceipts() (*types.Block, types.Receipts) { return nil, nil } +func (b *backendMock) GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) { + return nil, nil +} +func (b *backendMock) GetLogs(ctx context.Context, blockHash common.Hash, number uint64) ([][]*types.Log, error) { + return nil, nil +} +func (b *backendMock) GetTd(ctx context.Context, hash common.Hash) *big.Int { return nil } +func (b *backendMock) GetEVM(ctx context.Context, msg core.Message, state *state.StateDB, header *types.Header, vmConfig *vm.Config) (*vm.EVM, func() error, error) { + return nil, nil, nil +} +func (b *backendMock) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription { return nil } +func (b *backendMock) SubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) event.Subscription { + return nil +} +func (b *backendMock) SubscribeChainSideEvent(ch chan<- core.ChainSideEvent) event.Subscription { + return nil +} +func (b *backendMock) SendTx(ctx context.Context, signedTx *types.Transaction) error { return nil } +func (b *backendMock) GetTransaction(ctx context.Context, txHash common.Hash) (*types.Transaction, common.Hash, uint64, uint64, error) { + return nil, [32]byte{}, 0, 0, nil +} +func (b *backendMock) GetPoolTransactions() (types.Transactions, error) { return nil, nil } +func (b *backendMock) GetPoolTransaction(txHash common.Hash) *types.Transaction { return nil } +func (b *backendMock) GetPoolNonce(ctx context.Context, addr common.Address) (uint64, error) { + return 0, nil +} +func (b *backendMock) Stats() (pending int, queued int) { return 0, 0 } +func (b *backendMock) TxPoolContent() (map[common.Address]types.Transactions, map[common.Address]types.Transactions) { + return nil, nil +} +func (b *backendMock) TxPoolContentFrom(addr common.Address) (types.Transactions, types.Transactions) { + return nil, nil +} +func (b *backendMock) SubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscription { return nil } +func (b *backendMock) BloomStatus() (uint64, uint64) { return 0, 0 } +func (b *backendMock) ServiceFilter(ctx context.Context, session *bloombits.MatcherSession) {} +func (b *backendMock) SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription { return nil } +func (b *backendMock) SubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription { + return nil +} +func (b *backendMock) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription { + return nil +} + +func (b *backendMock) Engine() consensus.Engine { return nil } diff --git a/internal/flags/categories.go b/internal/flags/categories.go new file mode 100644 index 0000000000000..c2db6c6c1d25c --- /dev/null +++ b/internal/flags/categories.go @@ -0,0 +1,43 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package flags + +import "github.com/urfave/cli/v2" + +const ( + EthCategory = "ETHEREUM" + LightCategory = "LIGHT CLIENT" + DevCategory = "DEVELOPER CHAIN" + EthashCategory = "ETHASH" + TxPoolCategory = "TRANSACTION POOL" + PerfCategory = "PERFORMANCE TUNING" + AccountCategory = "ACCOUNT" + APICategory = "API AND CONSOLE" + NetworkingCategory = "NETWORKING" + MinerCategory = "MINER" + GasPriceCategory = "GAS PRICE ORACLE" + VMCategory = "VIRTUAL MACHINE" + LoggingCategory = "LOGGING AND DEBUGGING" + MetricsCategory = "METRICS AND STATS" + MiscCategory = "MISC" + DeprecatedCategory = "ALIASED (deprecated)" +) + +func init() { + cli.HelpFlag.(*cli.BoolFlag).Category = MiscCategory + cli.VersionFlag.(*cli.BoolFlag).Category = MiscCategory +} diff --git a/internal/flags/flags.go b/internal/flags/flags.go new file mode 100644 index 0000000000000..0ae2c6a512efe --- /dev/null +++ b/internal/flags/flags.go @@ -0,0 +1,340 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package flags + +import ( + "encoding" + "errors" + "flag" + "math/big" + "os" + "os/user" + "path" + "strings" + + "github.com/ethereum/go-ethereum/common/math" + "github.com/urfave/cli/v2" +) + +// DirectoryString is custom type which is registered in the flags library which cli uses for +// argument parsing. This allows us to expand Value to an absolute path when +// the argument is parsed +type DirectoryString string + +func (s *DirectoryString) String() string { + return string(*s) +} + +func (s *DirectoryString) Set(value string) error { + *s = DirectoryString(expandPath(value)) + return nil +} + +var ( + _ cli.Flag = (*DirectoryFlag)(nil) + _ cli.RequiredFlag = (*DirectoryFlag)(nil) + _ cli.VisibleFlag = (*DirectoryFlag)(nil) + _ cli.DocGenerationFlag = (*DirectoryFlag)(nil) + _ cli.CategorizableFlag = (*DirectoryFlag)(nil) +) + +// DirectoryFlag is custom cli.Flag type which expand the received string to an absolute path. +// e.g. ~/.ethereum -> /home/username/.ethereum +type DirectoryFlag struct { + Name string + + Category string + DefaultText string + Usage string + + Required bool + Hidden bool + HasBeenSet bool + + Value DirectoryString + + Aliases []string +} + +// For cli.Flag: + +func (f *DirectoryFlag) Names() []string { return append([]string{f.Name}, f.Aliases...) } +func (f *DirectoryFlag) IsSet() bool { return f.HasBeenSet } +func (f *DirectoryFlag) String() string { return cli.FlagStringer(f) } + +// Apply called by cli library, grabs variable from environment (if in env) +// and adds variable to flag set for parsing. +func (f *DirectoryFlag) Apply(set *flag.FlagSet) error { + eachName(f, func(name string) { + set.Var(&f.Value, f.Name, f.Usage) + }) + return nil +} + +// For cli.RequiredFlag: + +func (f *DirectoryFlag) IsRequired() bool { return f.Required } + +// For cli.VisibleFlag: + +func (f *DirectoryFlag) IsVisible() bool { return !f.Hidden } + +// For cli.CategorizableFlag: + +func (f *DirectoryFlag) GetCategory() string { return f.Category } + +// For cli.DocGenerationFlag: + +func (f *DirectoryFlag) TakesValue() bool { return true } +func (f *DirectoryFlag) GetUsage() string { return f.Usage } +func (f *DirectoryFlag) GetValue() string { return f.Value.String() } +func (f *DirectoryFlag) GetEnvVars() []string { return nil } // env not supported + +func (f *DirectoryFlag) GetDefaultText() string { + if f.DefaultText != "" { + return f.DefaultText + } + return f.GetValue() +} + +type TextMarshaler interface { + encoding.TextMarshaler + encoding.TextUnmarshaler +} + +// textMarshalerVal turns a TextMarshaler into a flag.Value +type textMarshalerVal struct { + v TextMarshaler +} + +func (v textMarshalerVal) String() string { + if v.v == nil { + return "" + } + text, _ := v.v.MarshalText() + return string(text) +} + +func (v textMarshalerVal) Set(s string) error { + return v.v.UnmarshalText([]byte(s)) +} + +var ( + _ cli.Flag = (*TextMarshalerFlag)(nil) + _ cli.RequiredFlag = (*TextMarshalerFlag)(nil) + _ cli.VisibleFlag = (*TextMarshalerFlag)(nil) + _ cli.DocGenerationFlag = (*TextMarshalerFlag)(nil) + _ cli.CategorizableFlag = (*TextMarshalerFlag)(nil) +) + +// TextMarshalerFlag wraps a TextMarshaler value. +type TextMarshalerFlag struct { + Name string + + Category string + DefaultText string + Usage string + + Required bool + Hidden bool + HasBeenSet bool + + Value TextMarshaler + + Aliases []string +} + +// For cli.Flag: + +func (f *TextMarshalerFlag) Names() []string { return append([]string{f.Name}, f.Aliases...) } +func (f *TextMarshalerFlag) IsSet() bool { return f.HasBeenSet } +func (f *TextMarshalerFlag) String() string { return cli.FlagStringer(f) } + +func (f *TextMarshalerFlag) Apply(set *flag.FlagSet) error { + eachName(f, func(name string) { + set.Var(textMarshalerVal{f.Value}, f.Name, f.Usage) + }) + return nil +} + +// For cli.RequiredFlag: + +func (f *TextMarshalerFlag) IsRequired() bool { return f.Required } + +// For cli.VisibleFlag: + +func (f *TextMarshalerFlag) IsVisible() bool { return !f.Hidden } + +// For cli.CategorizableFlag: + +func (f *TextMarshalerFlag) GetCategory() string { return f.Category } + +// For cli.DocGenerationFlag: + +func (f *TextMarshalerFlag) TakesValue() bool { return true } +func (f *TextMarshalerFlag) GetUsage() string { return f.Usage } +func (f *TextMarshalerFlag) GetEnvVars() []string { return nil } // env not supported + +func (f *TextMarshalerFlag) GetValue() string { + t, err := f.Value.MarshalText() + if err != nil { + return "(ERR: " + err.Error() + ")" + } + return string(t) +} + +func (f *TextMarshalerFlag) GetDefaultText() string { + if f.DefaultText != "" { + return f.DefaultText + } + return f.GetValue() +} + +// GlobalTextMarshaler returns the value of a TextMarshalerFlag from the global flag set. +func GlobalTextMarshaler(ctx *cli.Context, name string) TextMarshaler { + val := ctx.Generic(name) + if val == nil { + return nil + } + return val.(textMarshalerVal).v +} + +var ( + _ cli.Flag = (*BigFlag)(nil) + _ cli.RequiredFlag = (*BigFlag)(nil) + _ cli.VisibleFlag = (*BigFlag)(nil) + _ cli.DocGenerationFlag = (*BigFlag)(nil) + _ cli.CategorizableFlag = (*BigFlag)(nil) +) + +// BigFlag is a command line flag that accepts 256 bit big integers in decimal or +// hexadecimal syntax. +type BigFlag struct { + Name string + + Category string + DefaultText string + Usage string + + Required bool + Hidden bool + HasBeenSet bool + + Value *big.Int + + Aliases []string +} + +// For cli.Flag: + +func (f *BigFlag) Names() []string { return append([]string{f.Name}, f.Aliases...) } +func (f *BigFlag) IsSet() bool { return f.HasBeenSet } +func (f *BigFlag) String() string { return cli.FlagStringer(f) } + +func (f *BigFlag) Apply(set *flag.FlagSet) error { + eachName(f, func(name string) { + f.Value = new(big.Int) + set.Var((*bigValue)(f.Value), f.Name, f.Usage) + }) + + return nil +} + +// For cli.RequiredFlag: + +func (f *BigFlag) IsRequired() bool { return f.Required } + +// For cli.VisibleFlag: + +func (f *BigFlag) IsVisible() bool { return !f.Hidden } + +// For cli.CategorizableFlag: + +func (f *BigFlag) GetCategory() string { return f.Category } + +// For cli.DocGenerationFlag: + +func (f *BigFlag) TakesValue() bool { return true } +func (f *BigFlag) GetUsage() string { return f.Usage } +func (f *BigFlag) GetValue() string { return f.Value.String() } +func (f *BigFlag) GetEnvVars() []string { return nil } // env not supported + +func (f *BigFlag) GetDefaultText() string { + if f.DefaultText != "" { + return f.DefaultText + } + return f.GetValue() +} + +// bigValue turns *big.Int into a flag.Value +type bigValue big.Int + +func (b *bigValue) String() string { + if b == nil { + return "" + } + return (*big.Int)(b).String() +} + +func (b *bigValue) Set(s string) error { + intVal, ok := math.ParseBig256(s) + if !ok { + return errors.New("invalid integer syntax") + } + *b = (bigValue)(*intVal) + return nil +} + +// GlobalBig returns the value of a BigFlag from the global flag set. +func GlobalBig(ctx *cli.Context, name string) *big.Int { + val := ctx.Generic(name) + if val == nil { + return nil + } + return (*big.Int)(val.(*bigValue)) +} + +// Expands a file path +// 1. replace tilde with users home dir +// 2. expands embedded environment variables +// 3. cleans the path, e.g. /a/b/../c -> /a/c +// Note, it has limitations, e.g. ~someuser/tmp will not be expanded +func expandPath(p string) string { + if strings.HasPrefix(p, "~/") || strings.HasPrefix(p, "~\\") { + if home := HomeDir(); home != "" { + p = home + p[1:] + } + } + return path.Clean(os.ExpandEnv(p)) +} + +func HomeDir() string { + if home := os.Getenv("HOME"); home != "" { + return home + } + if usr, err := user.Current(); err == nil { + return usr.HomeDir + } + return "" +} + +func eachName(f cli.Flag, fn func(string)) { + for _, name := range f.Names() { + name = strings.Trim(name, " ") + fn(name) + } +} diff --git a/cmd/utils/customflags_test.go b/internal/flags/flags_test.go similarity index 61% rename from cmd/utils/customflags_test.go rename to internal/flags/flags_test.go index de39ca36a116c..a0d4af7ca3606 100644 --- a/cmd/utils/customflags_test.go +++ b/internal/flags/flags_test.go @@ -1,20 +1,20 @@ // Copyright 2015 The go-ethereum Authors -// This file is part of go-ethereum. +// This file is part of the go-ethereum library. // -// go-ethereum is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // -// go-ethereum is distributed in the hope that it will be useful, +// The go-ethereum library is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. +// GNU Lesser General Public License for more details. // -// You should have received a copy of the GNU General Public License -// along with go-ethereum. If not, see . +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . -package utils +package flags import ( "os" diff --git a/internal/flags/helpers.go b/internal/flags/helpers.go index 1fc6409c6550d..3be0a5807290e 100644 --- a/internal/flags/helpers.go +++ b/internal/flags/helpers.go @@ -17,137 +17,199 @@ package flags import ( - "os" - "path/filepath" + "fmt" + "strings" + "github.com/ethereum/go-ethereum/internal/version" "github.com/ethereum/go-ethereum/params" - "gopkg.in/urfave/cli.v1" + "github.com/urfave/cli/v2" ) -var ( - CommandHelpTemplate = `{{.cmd.Name}}{{if .cmd.Subcommands}} command{{end}}{{if .cmd.Flags}} [command options]{{end}} {{.cmd.ArgsUsage}} -{{if .cmd.Description}}{{.cmd.Description}} -{{end}}{{if .cmd.Subcommands}} -SUBCOMMANDS: - {{range .cmd.Subcommands}}{{.Name}}{{with .ShortName}}, {{.}}{{end}}{{ "\t" }}{{.Usage}} - {{end}}{{end}}{{if .categorizedFlags}} -{{range $idx, $categorized := .categorizedFlags}}{{$categorized.Name}} OPTIONS: -{{range $categorized.Flags}}{{"\t"}}{{.}} -{{end}} -{{end}}{{end}}` - - OriginCommandHelpTemplate = `{{.Name}}{{if .Subcommands}} command{{end}}{{if .Flags}} [command options]{{end}} {{.ArgsUsage}} -{{if .Description}}{{.Description}} -{{end}}{{if .Subcommands}} -SUBCOMMANDS: - {{range .Subcommands}}{{.Name}}{{with .ShortName}}, {{.}}{{end}}{{ "\t" }}{{.Usage}} - {{end}}{{end}}{{if .Flags}} -OPTIONS: -{{range $.Flags}} {{.}} -{{end}} -{{end}}` - - // AppHelpTemplate is the test template for the default, global app help topic. - AppHelpTemplate = `NAME: - {{.App.Name}} - {{.App.Usage}} - - Copyright 2013-2022 The go-ethereum Authors - -USAGE: - {{.App.HelpName}} [options]{{if .App.Commands}} [command] [command options]{{end}} {{if .App.ArgsUsage}}{{.App.ArgsUsage}}{{else}}[arguments...]{{end}} - {{if .App.Version}} -VERSION: - {{.App.Version}} - {{end}}{{if len .App.Authors}} -AUTHOR(S): - {{range .App.Authors}}{{ . }}{{end}} - {{end}}{{if .App.Commands}} -COMMANDS: - {{range .App.Commands}}{{join .Names ", "}}{{ "\t" }}{{.Usage}} - {{end}}{{end}}{{if .FlagGroups}} -{{range .FlagGroups}}{{.Name}} OPTIONS: - {{range .Flags}}{{.}} - {{end}} -{{end}}{{end}}{{if .App.Copyright }} -COPYRIGHT: - {{.App.Copyright}} - {{end}} -` - // ClefAppHelpTemplate is the template for the default, global app help topic. - ClefAppHelpTemplate = `NAME: - {{.App.Name}} - {{.App.Usage}} - - Copyright 2013-2022 The go-ethereum Authors - -USAGE: - {{.App.HelpName}} [options]{{if .App.Commands}} command [command options]{{end}} {{if .App.ArgsUsage}}{{.App.ArgsUsage}}{{else}}[arguments...]{{end}} - {{if .App.Version}} -COMMANDS: - {{range .App.Commands}}{{join .Names ", "}}{{ "\t" }}{{.Usage}} - {{end}}{{end}}{{if .FlagGroups}} -{{range .FlagGroups}}{{.Name}} OPTIONS: - {{range .Flags}}{{.}} - {{end}} -{{end}}{{end}}{{if .App.Copyright }} -COPYRIGHT: - {{.App.Copyright}} - {{end}} -` -) - -// HelpData is a one shot struct to pass to the usage template -type HelpData struct { - App interface{} - FlagGroups []FlagGroup +// NewApp creates an app with sane defaults. +func NewApp(usage string) *cli.App { + git, _ := version.VCS() + app := cli.NewApp() + app.EnableBashCompletion = true + app.Version = params.VersionWithCommit(git.Commit, git.Date) + app.Usage = usage + app.Copyright = "Copyright 2013-2022 The go-ethereum Authors" + app.Before = func(ctx *cli.Context) error { + MigrateGlobalFlags(ctx) + return nil + } + return app } -// FlagGroup is a collection of flags belonging to a single topic. -type FlagGroup struct { - Name string - Flags []cli.Flag +// Merge merges the given flag slices. +func Merge(groups ...[]cli.Flag) []cli.Flag { + var ret []cli.Flag + for _, group := range groups { + ret = append(ret, group...) + } + return ret } -// ByCategory sorts an array of FlagGroup by Name in the order -// defined in AppHelpFlagGroups. -type ByCategory []FlagGroup +var migrationApplied = map[*cli.Command]struct{}{} -func (a ByCategory) Len() int { return len(a) } -func (a ByCategory) Swap(i, j int) { a[i], a[j] = a[j], a[i] } -func (a ByCategory) Less(i, j int) bool { - iCat, jCat := a[i].Name, a[j].Name - iIdx, jIdx := len(a), len(a) // ensure non categorized flags come last +// MigrateGlobalFlags makes all global flag values available in the +// context. This should be called as early as possible in app.Before. +// +// Example: +// +// geth account new --keystore /tmp/mykeystore --lightkdf +// +// is equivalent after calling this method with: +// +// geth --keystore /tmp/mykeystore --lightkdf account new +// +// i.e. in the subcommand Action function of 'account new', ctx.Bool("lightkdf) +// will return true even if --lightkdf is set as a global option. +// +// This function may become unnecessary when https://github.com/urfave/cli/pull/1245 is merged. +func MigrateGlobalFlags(ctx *cli.Context) { + var iterate func(cs []*cli.Command, fn func(*cli.Command)) + iterate = func(cs []*cli.Command, fn func(*cli.Command)) { + for _, cmd := range cs { + if _, ok := migrationApplied[cmd]; ok { + continue + } + migrationApplied[cmd] = struct{}{} + fn(cmd) + iterate(cmd.Subcommands, fn) + } + } - for i, group := range a { - if iCat == group.Name { - iIdx = i + // This iterates over all commands and wraps their action function. + iterate(ctx.App.Commands, func(cmd *cli.Command) { + if cmd.Action == nil { + return } - if jCat == group.Name { - jIdx = i + + action := cmd.Action + cmd.Action = func(ctx *cli.Context) error { + doMigrateFlags(ctx) + return action(ctx) + } + }) +} + +func doMigrateFlags(ctx *cli.Context) { + // Figure out if there are any aliases of commands. If there are, we want + // to ignore them when iterating over the flags. + var aliases = make(map[string]bool) + for _, fl := range ctx.Command.Flags { + for _, alias := range fl.Names()[1:] { + aliases[alias] = true + } + } + for _, name := range ctx.FlagNames() { + for _, parent := range ctx.Lineage()[1:] { + if parent.IsSet(name) { + // When iterating across the lineage, we will be served both + // the 'canon' and alias formats of all commmands. In most cases, + // it's fine to set it in the ctx multiple times (one for each + // name), however, the Slice-flags are not fine. + // The slice-flags accumulate, so if we set it once as + // "foo" and once as alias "F", then both will be present in the slice. + if _, isAlias := aliases[name]; isAlias { + continue + } + // If it is a string-slice, we need to set it as + // "alfa, beta, gamma" instead of "[alfa beta gamma]", in order + // for the backing StringSlice to parse it properly. + if result := parent.StringSlice(name); len(result) > 0 { + ctx.Set(name, strings.Join(result, ",")) + } else { + ctx.Set(name, parent.String(name)) + } + break + } } } +} - return iIdx < jIdx +func init() { + cli.FlagStringer = FlagString } -func FlagCategory(flag cli.Flag, flagGroups []FlagGroup) string { - for _, category := range flagGroups { - for _, flg := range category.Flags { - if flg.GetName() == flag.GetName() { - return category.Name +// FlagString prints a single flag in help. +func FlagString(f cli.Flag) string { + df, ok := f.(cli.DocGenerationFlag) + if !ok { + return "" + } + + needsPlaceholder := df.TakesValue() + placeholder := "" + if needsPlaceholder { + placeholder = "value" + } + + namesText := pad(cli.FlagNamePrefixer(df.Names(), placeholder), 30) + + defaultValueString := "" + if s := df.GetDefaultText(); s != "" { + defaultValueString = " (default: " + s + ")" + } + + usage := strings.TrimSpace(df.GetUsage()) + envHint := strings.TrimSpace(cli.FlagEnvHinter(df.GetEnvVars(), "")) + if len(envHint) > 0 { + usage += " " + envHint + } + + usage = wordWrap(usage, 80) + usage = indent(usage, 10) + + return fmt.Sprintf("\n %s%s\n%s", namesText, defaultValueString, usage) +} + +func pad(s string, length int) string { + if len(s) < length { + s += strings.Repeat(" ", length-len(s)) + } + return s +} + +func indent(s string, nspace int) string { + ind := strings.Repeat(" ", nspace) + return ind + strings.ReplaceAll(s, "\n", "\n"+ind) +} + +func wordWrap(s string, width int) string { + var ( + output strings.Builder + lineLength = 0 + ) + + for { + sp := strings.IndexByte(s, ' ') + var word string + if sp == -1 { + word = s + } else { + word = s[:sp] + } + wlen := len(word) + over := lineLength+wlen >= width + if over { + output.WriteByte('\n') + lineLength = 0 + } else { + if lineLength != 0 { + output.WriteByte(' ') + lineLength++ } } + + output.WriteString(word) + lineLength += wlen + + if sp == -1 { + break + } + s = s[wlen+1:] } - return "MISC" -} -// NewApp creates an app with sane defaults. -func NewApp(gitCommit, gitDate, usage string) *cli.App { - app := cli.NewApp() - app.EnableBashCompletion = true - app.Name = filepath.Base(os.Args[0]) - app.Author = "" - app.Email = "" - app.Version = params.VersionWithCommit(gitCommit, gitDate) - app.Usage = usage - return app + return output.String() } diff --git a/internal/jsre/deps/web3.js b/internal/jsre/deps/web3.js index f82d93bdc570f..a291218ec51f9 100644 --- a/internal/jsre/deps/web3.js +++ b/internal/jsre/deps/web3.js @@ -3696,7 +3696,7 @@ var outputBigNumberFormatter = function (number) { }; var isPredefinedBlockNumber = function (blockNumber) { - return blockNumber === 'latest' || blockNumber === 'pending' || blockNumber === 'earliest' || blockNumber === 'finalized'; + return blockNumber === 'latest' || blockNumber === 'pending' || blockNumber === 'earliest' || blockNumber === 'finalized' || blockNumber === 'safe'; }; var inputDefaultBlockNumberFormatter = function (blockNumber) { diff --git a/internal/jsre/jsre.go b/internal/jsre/jsre.go index 4de80a9e901c0..f6e21d2ef7016 100644 --- a/internal/jsre/jsre.go +++ b/internal/jsre/jsre.go @@ -322,11 +322,11 @@ func (re *JSRE) loadScript(call Call) (goja.Value, error) { file = common.AbsolutePath(re.assetPath, file) source, err := os.ReadFile(file) if err != nil { - return nil, fmt.Errorf("Could not read file %s: %v", file, err) + return nil, fmt.Errorf("could not read file %s: %v", file, err) } value, err := compileAndRun(re.vm, file, string(source)) if err != nil { - return nil, fmt.Errorf("Error while compiling or running script: %v", err) + return nil, fmt.Errorf("error while compiling or running script: %v", err) } return value, nil } diff --git a/internal/jsre/pretty.go b/internal/jsre/pretty.go index 4171e00906173..bd772b4927c2d 100644 --- a/internal/jsre/pretty.go +++ b/internal/jsre/pretty.go @@ -219,7 +219,6 @@ func (ctx ppctx) fields(obj *goja.Object) []string { vals = append(vals, k) } } - } iterOwnAndConstructorKeys(ctx.vm, obj, add) sort.Strings(vals) diff --git a/internal/version/vcs_fallback.go b/internal/version/vcs_fallback.go new file mode 100644 index 0000000000000..f792c68cdb4c8 --- /dev/null +++ b/internal/version/vcs_fallback.go @@ -0,0 +1,28 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +//go:build !go1.18 +// +build !go1.18 + +package version + +import "runtime/debug" + +// In Go versions before 1.18, VCS information is not available. + +func buildInfoVCS(info *debug.BuildInfo) (VCSInfo, bool) { + return VCSInfo{}, false +} diff --git a/internal/version/vcs_go1.18.go b/internal/version/vcs_go1.18.go new file mode 100644 index 0000000000000..53cd41fb3097d --- /dev/null +++ b/internal/version/vcs_go1.18.go @@ -0,0 +1,55 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +//go:build go1.18 +// +build go1.18 + +package version + +import ( + "runtime/debug" + "time" +) + +// In go 1.18 and beyond, the go tool embeds VCS information into the build. + +const ( + govcsTimeLayout = "2006-01-02T15:04:05Z" + ourTimeLayout = "20060102" +) + +// buildInfoVCS returns VCS information of the build. +func buildInfoVCS(info *debug.BuildInfo) (s VCSInfo, ok bool) { + for _, v := range info.Settings { + switch v.Key { + case "vcs.revision": + s.Commit = v.Value + case "vcs.modified": + if v.Value == "true" { + s.Dirty = true + } + case "vcs.time": + t, err := time.Parse(govcsTimeLayout, v.Value) + if err == nil { + s.Date = t.Format(ourTimeLayout) + } + } + } + if s.Commit != "" && s.Date != "" { + ok = true + } + return +} diff --git a/internal/version/version.go b/internal/version/version.go new file mode 100644 index 0000000000000..4959102f7d840 --- /dev/null +++ b/internal/version/version.go @@ -0,0 +1,141 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Package version implements reading of build version information. +package version + +import ( + "fmt" + "runtime" + "runtime/debug" + "strings" + + "github.com/ethereum/go-ethereum/params" +) + +const ourPath = "github.com/ethereum/go-ethereum" // Path to our module + +// These variables are set at build-time by the linker when the build is +// done by build/ci.go. +var gitCommit, gitDate string + +// VCSInfo represents the git repository state. +type VCSInfo struct { + Commit string // head commit hash + Date string // commit time in YYYYMMDD format + Dirty bool +} + +// VCS returns version control information of the current executable. +func VCS() (VCSInfo, bool) { + if gitCommit != "" { + // Use information set by the build script if present. + return VCSInfo{Commit: gitCommit, Date: gitDate}, true + } + if buildInfo, ok := debug.ReadBuildInfo(); ok { + if buildInfo.Main.Path == ourPath { + return buildInfoVCS(buildInfo) + } + } + return VCSInfo{}, false +} + +// ClientName creates a software name/version identifier according to common +// conventions in the Ethereum p2p network. +func ClientName(clientIdentifier string) string { + git, _ := VCS() + return fmt.Sprintf("%s/v%v/%v-%v/%v", + strings.Title(clientIdentifier), + params.VersionWithCommit(git.Commit, git.Date), + runtime.GOOS, runtime.GOARCH, + runtime.Version(), + ) +} + +// runtimeInfo returns build and platform information about the current binary. +// +// If the package that is currently executing is a prefixed by our go-ethereum +// module path, it will print out commit and date VCS information. Otherwise, +// it will assume it's imported by a third-party and will return the imported +// version and whether it was replaced by another module. +func Info() (version, vcs string) { + version = params.VersionWithMeta + buildInfo, ok := debug.ReadBuildInfo() + if !ok { + return version, "" + } + version = versionInfo(buildInfo) + if status, ok := VCS(); ok { + modified := "" + if status.Dirty { + modified = " (dirty)" + } + commit := status.Commit + if len(commit) > 8 { + commit = commit[:8] + } + vcs = commit + "-" + status.Date + modified + } + return version, vcs +} + +// versionInfo returns version information for the currently executing +// implementation. +// +// Depending on how the code is instansiated, it returns different amounts of +// information. If it is unable to determine which module is related to our +// package it falls back to the hardcoded values in the params package. +func versionInfo(info *debug.BuildInfo) string { + // If the main package is from our repo, prefix version with "geth". + if strings.HasPrefix(info.Path, ourPath) { + return fmt.Sprintf("geth %s", info.Main.Version) + } + // Not our main package, so explicitly print out the module path and + // version. + var version string + if info.Main.Path != "" && info.Main.Version != "" { + // These can be empty when invoked with "go run". + version = fmt.Sprintf("%s@%s ", info.Main.Path, info.Main.Version) + } + mod := findModule(info, ourPath) + if mod == nil { + // If our module path wasn't imported, it's unclear which + // version of our code they are running. Fallback to hardcoded + // version. + return version + fmt.Sprintf("geth %s", params.VersionWithMeta) + } + // Our package is a dependency for the main module. Return path and + // version data for both. + version += fmt.Sprintf("%s@%s", mod.Path, mod.Version) + if mod.Replace != nil { + // If our package was replaced by something else, also note that. + version += fmt.Sprintf(" (replaced by %s@%s)", mod.Replace.Path, mod.Replace.Version) + } + return version +} + +// findModule returns the module at path. +func findModule(info *debug.BuildInfo, path string) *debug.Module { + if info.Path == ourPath { + return &info.Main + } + for _, mod := range info.Deps { + if mod.Path == path { + return mod + } + } + return nil +} diff --git a/internal/web3ext/web3ext.go b/internal/web3ext/web3ext.go index 88c31c04da192..134562bde6fc1 100644 --- a/internal/web3ext/web3ext.go +++ b/internal/web3ext/web3ext.go @@ -224,13 +224,13 @@ web3._extend({ outputFormatter: console.log }), new web3._extend.Method({ - name: 'getHeaderRlp', - call: 'debug_getHeaderRlp', + name: 'getRawHeader', + call: 'debug_getRawHeader', params: 1 }), new web3._extend.Method({ - name: 'getBlockRlp', - call: 'debug_getBlockRlp', + name: 'getRawBlock', + call: 'debug_getRawBlock', params: 1 }), new web3._extend.Method({ @@ -238,6 +238,11 @@ web3._extend({ call: 'debug_getRawReceipts', params: 1 }), + new web3._extend.Method({ + name: 'getRawTransaction', + call: 'debug_getRawTransaction', + params: 1 + }), new web3._extend.Method({ name: 'setHead', call: 'debug_setHead', diff --git a/les/api.go b/les/api.go index 782bb31ef29a0..76714baef0869 100644 --- a/les/api.go +++ b/les/api.go @@ -33,15 +33,15 @@ var ( errUnknownBenchmarkType = errors.New("unknown benchmark type") ) -// PrivateLightServerAPI provides an API to access the LES light server. -type PrivateLightServerAPI struct { +// LightServerAPI provides an API to access the LES light server. +type LightServerAPI struct { server *LesServer defaultPosFactors, defaultNegFactors vfs.PriceFactors } -// NewPrivateLightServerAPI creates a new LES light server API. -func NewPrivateLightServerAPI(server *LesServer) *PrivateLightServerAPI { - return &PrivateLightServerAPI{ +// NewLightServerAPI creates a new LES light server API. +func NewLightServerAPI(server *LesServer) *LightServerAPI { + return &LightServerAPI{ server: server, defaultPosFactors: defaultPosFactors, defaultNegFactors: defaultNegFactors, @@ -61,7 +61,7 @@ func parseNode(node string) (enode.ID, error) { } // ServerInfo returns global server parameters -func (api *PrivateLightServerAPI) ServerInfo() map[string]interface{} { +func (api *LightServerAPI) ServerInfo() map[string]interface{} { res := make(map[string]interface{}) res["minimumCapacity"] = api.server.minCapacity res["maximumCapacity"] = api.server.maxCapacity @@ -72,7 +72,7 @@ func (api *PrivateLightServerAPI) ServerInfo() map[string]interface{} { } // ClientInfo returns information about clients listed in the ids list or matching the given tags -func (api *PrivateLightServerAPI) ClientInfo(nodes []string) map[enode.ID]map[string]interface{} { +func (api *LightServerAPI) ClientInfo(nodes []string) map[enode.ID]map[string]interface{} { var ids []enode.ID for _, node := range nodes { if id, err := parseNode(node); err == nil { @@ -102,7 +102,7 @@ func (api *PrivateLightServerAPI) ClientInfo(nodes []string) map[enode.ID]map[st // If maxCount limit is applied but there are more potential results then the ID // of the next potential result is included in the map with an empty structure // assigned to it. -func (api *PrivateLightServerAPI) PriorityClientInfo(start, stop enode.ID, maxCount int) map[enode.ID]map[string]interface{} { +func (api *LightServerAPI) PriorityClientInfo(start, stop enode.ID, maxCount int) map[enode.ID]map[string]interface{} { res := make(map[enode.ID]map[string]interface{}) ids := api.server.clientPool.GetPosBalanceIDs(start, stop, maxCount+1) if len(ids) > maxCount { @@ -122,7 +122,7 @@ func (api *PrivateLightServerAPI) PriorityClientInfo(start, stop enode.ID, maxCo } // clientInfo creates a client info data structure -func (api *PrivateLightServerAPI) clientInfo(peer *clientPeer, balance vfs.ReadOnlyBalance) map[string]interface{} { +func (api *LightServerAPI) clientInfo(peer *clientPeer, balance vfs.ReadOnlyBalance) map[string]interface{} { info := make(map[string]interface{}) pb, nb := balance.GetBalance() info["isConnected"] = peer != nil @@ -140,7 +140,7 @@ func (api *PrivateLightServerAPI) clientInfo(peer *clientPeer, balance vfs.ReadO // setParams either sets the given parameters for a single connected client (if specified) // or the default parameters applicable to clients connected in the future -func (api *PrivateLightServerAPI) setParams(params map[string]interface{}, client *clientPeer, posFactors, negFactors *vfs.PriceFactors) (updateFactors bool, err error) { +func (api *LightServerAPI) setParams(params map[string]interface{}, client *clientPeer, posFactors, negFactors *vfs.PriceFactors) (updateFactors bool, err error) { defParams := client == nil for name, value := range params { errValue := func() error { @@ -191,7 +191,7 @@ func (api *PrivateLightServerAPI) setParams(params map[string]interface{}, clien // SetClientParams sets client parameters for all clients listed in the ids list // or all connected clients if the list is empty -func (api *PrivateLightServerAPI) SetClientParams(nodes []string, params map[string]interface{}) error { +func (api *LightServerAPI) SetClientParams(nodes []string, params map[string]interface{}) error { var err error for _, node := range nodes { var id enode.ID @@ -215,7 +215,7 @@ func (api *PrivateLightServerAPI) SetClientParams(nodes []string, params map[str } // SetDefaultParams sets the default parameters applicable to clients connected in the future -func (api *PrivateLightServerAPI) SetDefaultParams(params map[string]interface{}) error { +func (api *LightServerAPI) SetDefaultParams(params map[string]interface{}) error { update, err := api.setParams(params, nil, &api.defaultPosFactors, &api.defaultNegFactors) if update { api.server.clientPool.SetDefaultFactors(api.defaultPosFactors, api.defaultNegFactors) @@ -227,7 +227,7 @@ func (api *PrivateLightServerAPI) SetDefaultParams(params map[string]interface{} // So that already connected client won't be kicked out very soon and we can ensure all // connected clients can have enough time to request or sync some data. // When the input parameter `bias` < 0 (illegal), return error. -func (api *PrivateLightServerAPI) SetConnectedBias(bias time.Duration) error { +func (api *LightServerAPI) SetConnectedBias(bias time.Duration) error { if bias < time.Duration(0) { return fmt.Errorf("bias illegal: %v less than 0", bias) } @@ -237,7 +237,7 @@ func (api *PrivateLightServerAPI) SetConnectedBias(bias time.Duration) error { // AddBalance adds the given amount to the balance of a client if possible and returns // the balance before and after the operation -func (api *PrivateLightServerAPI) AddBalance(node string, amount int64) (balance [2]uint64, err error) { +func (api *LightServerAPI) AddBalance(node string, amount int64) (balance [2]uint64, err error) { var id enode.ID if id, err = parseNode(node); err != nil { return @@ -254,7 +254,7 @@ func (api *PrivateLightServerAPI) AddBalance(node string, amount int64) (balance // // Note: measurement time is adjusted for each pass depending on the previous ones. // Therefore a controlled total measurement time is achievable in multiple passes. -func (api *PrivateLightServerAPI) Benchmark(setups []map[string]interface{}, passCount, length int) ([]map[string]interface{}, error) { +func (api *LightServerAPI) Benchmark(setups []map[string]interface{}, passCount, length int) ([]map[string]interface{}, error) { benchmarks := make([]requestBenchmark, len(setups)) for i, setup := range setups { if t, ok := setup["type"].(string); ok { @@ -324,20 +324,20 @@ func (api *PrivateLightServerAPI) Benchmark(setups []map[string]interface{}, pas return result, nil } -// PrivateDebugAPI provides an API to debug LES light server functionality. -type PrivateDebugAPI struct { +// DebugAPI provides an API to debug LES light server functionality. +type DebugAPI struct { server *LesServer } -// NewPrivateDebugAPI creates a new LES light server debug API. -func NewPrivateDebugAPI(server *LesServer) *PrivateDebugAPI { - return &PrivateDebugAPI{ +// NewDebugAPI creates a new LES light server debug API. +func NewDebugAPI(server *LesServer) *DebugAPI { + return &DebugAPI{ server: server, } } // FreezeClient forces a temporary client freeze which normally happens when the server is overloaded -func (api *PrivateDebugAPI) FreezeClient(node string) error { +func (api *DebugAPI) FreezeClient(node string) error { var ( id enode.ID err error @@ -353,24 +353,25 @@ func (api *PrivateDebugAPI) FreezeClient(node string) error { } } -// PrivateLightAPI provides an API to access the LES light server or light client. -type PrivateLightAPI struct { +// LightAPI provides an API to access the LES light server or light client. +type LightAPI struct { backend *lesCommons } -// NewPrivateLightAPI creates a new LES service API. -func NewPrivateLightAPI(backend *lesCommons) *PrivateLightAPI { - return &PrivateLightAPI{backend: backend} +// NewLightAPI creates a new LES service API. +func NewLightAPI(backend *lesCommons) *LightAPI { + return &LightAPI{backend: backend} } // LatestCheckpoint returns the latest local checkpoint package. // // The checkpoint package consists of 4 strings: -// result[0], hex encoded latest section index -// result[1], 32 bytes hex encoded latest section head hash -// result[2], 32 bytes hex encoded latest section canonical hash trie root hash -// result[3], 32 bytes hex encoded latest section bloom trie root hash -func (api *PrivateLightAPI) LatestCheckpoint() ([4]string, error) { +// +// result[0], hex encoded latest section index +// result[1], 32 bytes hex encoded latest section head hash +// result[2], 32 bytes hex encoded latest section canonical hash trie root hash +// result[3], 32 bytes hex encoded latest section bloom trie root hash +func (api *LightAPI) LatestCheckpoint() ([4]string, error) { var res [4]string cp := api.backend.latestLocalCheckpoint() if cp.Empty() { @@ -384,10 +385,11 @@ func (api *PrivateLightAPI) LatestCheckpoint() ([4]string, error) { // GetLocalCheckpoint returns the specific local checkpoint package. // // The checkpoint package consists of 3 strings: -// result[0], 32 bytes hex encoded latest section head hash -// result[1], 32 bytes hex encoded latest section canonical hash trie root hash -// result[2], 32 bytes hex encoded latest section bloom trie root hash -func (api *PrivateLightAPI) GetCheckpoint(index uint64) ([3]string, error) { +// +// result[0], 32 bytes hex encoded latest section head hash +// result[1], 32 bytes hex encoded latest section canonical hash trie root hash +// result[2], 32 bytes hex encoded latest section bloom trie root hash +func (api *LightAPI) GetCheckpoint(index uint64) ([3]string, error) { var res [3]string cp := api.backend.localCheckpoint(index) if cp.Empty() { @@ -398,7 +400,7 @@ func (api *PrivateLightAPI) GetCheckpoint(index uint64) ([3]string, error) { } // GetCheckpointContractAddress returns the contract contract address in hex format. -func (api *PrivateLightAPI) GetCheckpointContractAddress() (string, error) { +func (api *LightAPI) GetCheckpointContractAddress() (string, error) { if api.backend.oracle == nil { return "", errNotActivated } diff --git a/les/api_backend.go b/les/api_backend.go index 11a9ca128aab5..71cfbbed1e55d 100644 --- a/les/api_backend.go +++ b/les/api_backend.go @@ -33,6 +33,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/eth/gasprice" + "github.com/ethereum/go-ethereum/eth/tracers" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/light" @@ -168,11 +169,8 @@ func (b *LesApiBackend) GetReceipts(ctx context.Context, hash common.Hash) (type return nil, nil } -func (b *LesApiBackend) GetLogs(ctx context.Context, hash common.Hash) ([][]*types.Log, error) { - if number := rawdb.ReadHeaderNumber(b.eth.chainDb, hash); number != nil { - return light.GetBlockLogs(ctx, b.eth.odr, hash, *number) - } - return nil, nil +func (b *LesApiBackend) GetLogs(ctx context.Context, hash common.Hash, number uint64) ([][]*types.Log, error) { + return light.GetBlockLogs(ctx, b.eth.odr, hash, number) } func (b *LesApiBackend) GetTd(ctx context.Context, hash common.Hash) *big.Int { @@ -324,10 +322,10 @@ func (b *LesApiBackend) CurrentHeader() *types.Header { return b.eth.blockchain.CurrentHeader() } -func (b *LesApiBackend) StateAtBlock(ctx context.Context, block *types.Block, reexec uint64, base *state.StateDB, checkLive bool, preferDisk bool) (*state.StateDB, error) { +func (b *LesApiBackend) StateAtBlock(ctx context.Context, block *types.Block, reexec uint64, base *state.StateDB, readOnly bool, preferDisk bool) (*state.StateDB, tracers.StateReleaseFunc, error) { return b.eth.stateAtBlock(ctx, block, reexec) } -func (b *LesApiBackend) StateAtTransaction(ctx context.Context, block *types.Block, txIndex int, reexec uint64) (core.Message, vm.BlockContext, *state.StateDB, error) { +func (b *LesApiBackend) StateAtTransaction(ctx context.Context, block *types.Block, txIndex int, reexec uint64) (core.Message, vm.BlockContext, *state.StateDB, tracers.StateReleaseFunc, error) { return b.eth.stateAtTransaction(ctx, block, txIndex, reexec) } diff --git a/les/api_test.go b/les/api_test.go index ea6870e356275..3db1c5fd5ec9c 100644 --- a/les/api_test.go +++ b/les/api_test.go @@ -340,7 +340,6 @@ func freezeClient(ctx context.Context, t *testing.T, server *rpc.Client, clientI if err := server.CallContext(ctx, nil, "debug_freezeClient", clientID); err != nil { t.Fatalf("Failed to freeze client: %v", err) } - } func setCapacity(ctx context.Context, t *testing.T, server *rpc.Client, clientID enode.ID, cap uint64) { diff --git a/les/catalyst/api.go b/les/catalyst/api.go index de09acdb02133..822e0af038a7e 100644 --- a/les/catalyst/api.go +++ b/les/catalyst/api.go @@ -22,6 +22,7 @@ import ( "fmt" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/beacon" "github.com/ethereum/go-ethereum/les" "github.com/ethereum/go-ethereum/log" @@ -35,9 +36,7 @@ func Register(stack *node.Node, backend *les.LightEthereum) error { stack.RegisterAPIs([]rpc.API{ { Namespace: "engine", - Version: "1.0", Service: NewConsensusAPI(backend), - Public: true, Authenticated: true, }, }) @@ -52,21 +51,25 @@ type ConsensusAPI struct { // The underlying blockchain needs to have a valid terminal total difficulty set. func NewConsensusAPI(les *les.LightEthereum) *ConsensusAPI { if les.BlockChain().Config().TerminalTotalDifficulty == nil { - panic("Catalyst started without valid total difficulty") + log.Warn("Catalyst started without valid total difficulty") } return &ConsensusAPI{les: les} } // ForkchoiceUpdatedV1 has several responsibilities: -// If the method is called with an empty head block: -// we return success, which can be used to check if the catalyst mode is enabled -// If the total difficulty was not reached: -// we return INVALID -// If the finalizedBlockHash is set: -// we check if we have the finalizedBlockHash in our db, if not we start a sync -// We try to set our blockchain to the headBlock -// If there are payloadAttributes: -// we return an error since block creation is not supported in les mode +// +// We try to set our blockchain to the headBlock. +// +// If the method is called with an empty head block: we return success, which can be used +// to check if the catalyst mode is enabled. +// +// If the total difficulty was not reached: we return INVALID. +// +// If the finalizedBlockHash is set: we check if we have the finalizedBlockHash in our db, +// if not we start a sync. +// +// If there are payloadAttributes: we return an error since block creation is not +// supported in les mode. func (api *ConsensusAPI) ForkchoiceUpdatedV1(heads beacon.ForkchoiceStateV1, payloadAttributes *beacon.PayloadAttributesV1) (beacon.ForkChoiceResponse, error) { if heads.HeadBlockHash == (common.Hash{}) { log.Warn("Forkchoice requested update to zero hash") @@ -187,3 +190,31 @@ func (api *ConsensusAPI) setCanonical(newHead common.Hash) error { } return nil } + +// ExchangeTransitionConfigurationV1 checks the given configuration against +// the configuration of the node. +func (api *ConsensusAPI) ExchangeTransitionConfigurationV1(config beacon.TransitionConfigurationV1) (*beacon.TransitionConfigurationV1, error) { + log.Trace("Engine API request received", "method", "ExchangeTransitionConfiguration", "ttd", config.TerminalTotalDifficulty) + if config.TerminalTotalDifficulty == nil { + return nil, errors.New("invalid terminal total difficulty") + } + + ttd := api.les.BlockChain().Config().TerminalTotalDifficulty + if ttd == nil || ttd.Cmp(config.TerminalTotalDifficulty.ToInt()) != 0 { + log.Warn("Invalid TTD configured", "geth", ttd, "beacon", config.TerminalTotalDifficulty) + return nil, fmt.Errorf("invalid ttd: execution %v consensus %v", ttd, config.TerminalTotalDifficulty) + } + + if config.TerminalBlockHash != (common.Hash{}) { + if hash := api.les.BlockChain().GetCanonicalHash(uint64(config.TerminalBlockNumber)); hash == config.TerminalBlockHash { + return &beacon.TransitionConfigurationV1{ + TerminalTotalDifficulty: (*hexutil.Big)(ttd), + TerminalBlockHash: config.TerminalBlockHash, + TerminalBlockNumber: config.TerminalBlockNumber, + }, nil + } + return nil, fmt.Errorf("invalid terminal block hash") + } + + return &beacon.TransitionConfigurationV1{TerminalTotalDifficulty: (*hexutil.Big)(ttd)}, nil +} diff --git a/les/catalyst/api_test.go b/les/catalyst/api_test.go index 70a6d24719ea3..6d0eedeccb774 100644 --- a/les/catalyst/api_test.go +++ b/les/catalyst/api_test.go @@ -24,7 +24,6 @@ import ( "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/beacon" - "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth/downloader" @@ -46,18 +45,15 @@ var ( ) func generatePreMergeChain(n int) (*core.Genesis, []*types.Header, []*types.Block) { - db := rawdb.NewMemoryDatabase() - config := params.AllEthashProtocolChanges + config := *params.AllEthashProtocolChanges genesis := &core.Genesis{ - Config: config, + Config: &config, Alloc: core.GenesisAlloc{testAddr: {Balance: testBalance}}, ExtraData: []byte("test genesis"), Timestamp: 9000, BaseFee: big.NewInt(params.InitialBaseFee), } - gblock := genesis.ToBlock(db) - engine := ethash.NewFaker() - blocks, _ := core.GenerateChain(config, gblock, engine, db, n, nil) + _, blocks, _ := core.GenerateChainWithGenesis(genesis, ethash.NewFaker(), n, nil) totalDifficulty := big.NewInt(0) var headers []*types.Header diff --git a/les/client.go b/les/client.go index c3acbc2e4bf8a..c304bf86f8a83 100644 --- a/les/client.go +++ b/les/client.go @@ -19,6 +19,7 @@ package les import ( "fmt" + "strings" "time" "github.com/ethereum/go-ethereum/accounts" @@ -31,7 +32,6 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth/ethconfig" - "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/eth/gasprice" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/internal/ethapi" @@ -73,7 +73,7 @@ type LightEthereum struct { eventMux *event.TypeMux engine consensus.Engine accountManager *accounts.Manager - netRPCService *ethapi.PublicNetAPI + netRPCService *ethapi.NetAPI p2pServer *p2p.Server p2pConfig *p2p.Config @@ -92,11 +92,24 @@ func New(stack *node.Node, config *ethconfig.Config) (*LightEthereum, error) { if err != nil { return nil, err } - chainConfig, genesisHash, genesisErr := core.SetupGenesisBlockWithOverride(chainDb, config.Genesis, config.OverrideArrowGlacier, config.OverrideTerminalTotalDifficulty) + var overrides core.ChainOverrides + if config.OverrideTerminalTotalDifficulty != nil { + overrides.OverrideTerminalTotalDifficulty = config.OverrideTerminalTotalDifficulty + } + if config.OverrideTerminalTotalDifficultyPassed != nil { + overrides.OverrideTerminalTotalDifficultyPassed = config.OverrideTerminalTotalDifficultyPassed + } + chainConfig, genesisHash, genesisErr := core.SetupGenesisBlockWithOverride(chainDb, config.Genesis, &overrides) if _, isCompat := genesisErr.(*params.ConfigCompatError); genesisErr != nil && !isCompat { return nil, genesisErr } - log.Info("Initialised chain configuration", "config", chainConfig) + log.Info("") + log.Info(strings.Repeat("-", 153)) + for _, line := range strings.Split(chainConfig.Description(), "\n") { + log.Info(line) + } + log.Info(strings.Repeat("-", 153)) + log.Info("") peers := newServerPeerSet() merger := consensus.NewMerger(chainDb) @@ -115,7 +128,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*LightEthereum, error) { reqDist: newRequestDistributor(peers, &mclock.System{}), accountManager: stack.AccountManager(), merger: merger, - engine: ethconfig.CreateConsensusEngine(stack, chainConfig, &config.Ethash, nil, false, chainDb), + engine: ethconfig.CreateConsensusEngine(stack, &config.Ethash, chainConfig.Clique, nil, false, chainDb), bloomRequests: make(chan chan *bloombits.Retrieval), bloomIndexer: core.NewBloomIndexer(chainDb, params.BloomBitsBlocksClient, params.HelperTrieConfirmations), p2pServer: stack.Server(), @@ -182,7 +195,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*LightEthereum, error) { leth.blockchain.DisableCheckFreq() } - leth.netRPCService = ethapi.NewPublicNetAPI(leth.p2pServer, leth.config.NetworkId) + leth.netRPCService = ethapi.NewNetAPI(leth.p2pServer, leth.config.NetworkId) // Register the backend on the node stack.RegisterAPIs(leth.APIs()) @@ -287,34 +300,19 @@ func (s *LightEthereum) APIs() []rpc.API { return append(apis, []rpc.API{ { Namespace: "eth", - Version: "1.0", Service: &LightDummyAPI{}, - Public: true, - }, { - Namespace: "eth", - Version: "1.0", - Service: downloader.NewPublicDownloaderAPI(s.handler.downloader, s.eventMux), - Public: true, }, { Namespace: "eth", - Version: "1.0", - Service: filters.NewPublicFilterAPI(s.ApiBackend, true, 5*time.Minute), - Public: true, + Service: downloader.NewDownloaderAPI(s.handler.downloader, s.eventMux), }, { Namespace: "net", - Version: "1.0", Service: s.netRPCService, - Public: true, }, { Namespace: "les", - Version: "1.0", - Service: NewPrivateLightAPI(&s.lesCommons), - Public: false, + Service: NewLightAPI(&s.lesCommons), }, { Namespace: "vflux", - Version: "1.0", Service: s.serverPool.API(), - Public: false, }, }...) } diff --git a/les/commons.go b/les/commons.go index d090fc21fccaf..e83319543d36b 100644 --- a/les/commons.go +++ b/les/commons.go @@ -63,7 +63,7 @@ type lesCommons struct { // NodeInfo represents a short summary of the Ethereum sub-protocol metadata // known about the host peer. type NodeInfo struct { - Network uint64 `json:"network"` // Ethereum network ID (1=Frontier, 2=Morden, Ropsten=3, Rinkeby=4) + Network uint64 `json:"network"` // Ethereum network ID (1=Mainnet, Ropsten=3, Rinkeby=4, Goerli=5) Difficulty *big.Int `json:"difficulty"` // Total difficulty of the host's blockchain Genesis common.Hash `json:"genesis"` // SHA3 hash of the host's genesis block Config *params.ChainConfig `json:"config"` // Chain configuration for the fork rules diff --git a/les/distributor.go b/les/distributor.go index 31150e4d731a7..a0319c67f7373 100644 --- a/les/distributor.go +++ b/les/distributor.go @@ -256,7 +256,7 @@ func (d *requestDistributor) queue(r *distReq) chan distPeer { if r.reqOrder == 0 { d.lastReqOrder++ r.reqOrder = d.lastReqOrder - r.waitForPeers = d.clock.Now() + mclock.AbsTime(waitForPeers) + r.waitForPeers = d.clock.Now().Add(waitForPeers) } // Assign the timestamp when the request is queued no matter it's // a new one or re-queued one. diff --git a/les/downloader/api.go b/les/downloader/api.go index 2024d23deade6..21200b676c644 100644 --- a/les/downloader/api.go +++ b/les/downloader/api.go @@ -25,21 +25,21 @@ import ( "github.com/ethereum/go-ethereum/rpc" ) -// PublicDownloaderAPI provides an API which gives information about the current synchronisation status. +// DownloaderAPI provides an API which gives information about the current synchronisation status. // It offers only methods that operates on data that can be available to anyone without security risks. -type PublicDownloaderAPI struct { +type DownloaderAPI struct { d *Downloader mux *event.TypeMux installSyncSubscription chan chan interface{} uninstallSyncSubscription chan *uninstallSyncSubscriptionRequest } -// NewPublicDownloaderAPI create a new PublicDownloaderAPI. The API has an internal event loop that +// NewDownloaderAPI create a new PublicDownloaderAPI. The API has an internal event loop that // listens for events from the downloader through the global event mux. In case it receives one of // these events it broadcasts it to all syncing subscriptions that are installed through the // installSyncSubscription channel. -func NewPublicDownloaderAPI(d *Downloader, m *event.TypeMux) *PublicDownloaderAPI { - api := &PublicDownloaderAPI{ +func NewDownloaderAPI(d *Downloader, m *event.TypeMux) *DownloaderAPI { + api := &DownloaderAPI{ d: d, mux: m, installSyncSubscription: make(chan chan interface{}), @@ -53,7 +53,7 @@ func NewPublicDownloaderAPI(d *Downloader, m *event.TypeMux) *PublicDownloaderAP // eventLoop runs a loop until the event mux closes. It will install and uninstall new // sync subscriptions and broadcasts sync status updates to the installed sync subscriptions. -func (api *PublicDownloaderAPI) eventLoop() { +func (api *DownloaderAPI) eventLoop() { var ( sub = api.mux.Subscribe(StartEvent{}, DoneEvent{}, FailedEvent{}) syncSubscriptions = make(map[chan interface{}]struct{}) @@ -90,7 +90,7 @@ func (api *PublicDownloaderAPI) eventLoop() { } // Syncing provides information when this nodes starts synchronising with the Ethereum network and when it's finished. -func (api *PublicDownloaderAPI) Syncing(ctx context.Context) (*rpc.Subscription, error) { +func (api *DownloaderAPI) Syncing(ctx context.Context) (*rpc.Subscription, error) { notifier, supported := rpc.NotifierFromContext(ctx) if !supported { return &rpc.Subscription{}, rpc.ErrNotificationsUnsupported @@ -125,7 +125,7 @@ type SyncingResult struct { Status ethereum.SyncProgress `json:"status"` } -// uninstallSyncSubscriptionRequest uninstalles a syncing subscription in the API event loop. +// uninstallSyncSubscriptionRequest uninstalls a syncing subscription in the API event loop. type uninstallSyncSubscriptionRequest struct { c chan interface{} uninstalled chan interface{} @@ -133,9 +133,9 @@ type uninstallSyncSubscriptionRequest struct { // SyncStatusSubscription represents a syncing subscription. type SyncStatusSubscription struct { - api *PublicDownloaderAPI // register subscription in event loop of this api instance - c chan interface{} // channel where events are broadcasted to - unsubOnce sync.Once // make sure unsubscribe logic is executed once + api *DownloaderAPI // register subscription in event loop of this api instance + c chan interface{} // channel where events are broadcasted to + unsubOnce sync.Once // make sure unsubscribe logic is executed once } // Unsubscribe uninstalls the subscription from the DownloadAPI event loop. @@ -160,7 +160,7 @@ func (s *SyncStatusSubscription) Unsubscribe() { // SubscribeSyncStatus creates a subscription that will broadcast new synchronisation updates. // The given channel must receive interface values, the result can either -func (api *PublicDownloaderAPI) SubscribeSyncStatus(status chan interface{}) *SyncStatusSubscription { +func (api *DownloaderAPI) SubscribeSyncStatus(status chan interface{}) *SyncStatusSubscription { api.installSyncSubscription <- status return &SyncStatusSubscription{api: api, c: status} } diff --git a/les/downloader/downloader.go b/les/downloader/downloader.go index 448a94192b876..740fdbdad1a3b 100644 --- a/les/downloader/downloader.go +++ b/les/downloader/downloader.go @@ -693,9 +693,11 @@ func (d *Downloader) fetchHead(p *peerConnection) (head *types.Header, pivot *ty // calculateRequestSpan calculates what headers to request from a peer when trying to determine the // common ancestor. // It returns parameters to be used for peer.RequestHeadersByNumber: -// from - starting block number -// count - number of headers to request -// skip - number of headers to skip +// +// from - starting block number +// count - number of headers to request +// skip - number of headers to skip +// // and also returns 'max', the last block which is expected to be returned by the remote peers, // given the (from,count,skip) func calculateRequestSpan(remoteHeight, localHeight uint64) (int64, int, int, uint64) { @@ -1310,27 +1312,26 @@ func (d *Downloader) fetchReceipts(from uint64) error { // various callbacks to handle the slight differences between processing them. // // The instrumentation parameters: -// - errCancel: error type to return if the fetch operation is cancelled (mostly makes logging nicer) -// - deliveryCh: channel from which to retrieve downloaded data packets (merged from all concurrent peers) -// - deliver: processing callback to deliver data packets into type specific download queues (usually within `queue`) -// - wakeCh: notification channel for waking the fetcher when new tasks are available (or sync completed) -// - expire: task callback method to abort requests that took too long and return the faulty peers (traffic shaping) -// - pending: task callback for the number of requests still needing download (detect completion/non-completability) -// - inFlight: task callback for the number of in-progress requests (wait for all active downloads to finish) -// - throttle: task callback to check if the processing queue is full and activate throttling (bound memory use) -// - reserve: task callback to reserve new download tasks to a particular peer (also signals partial completions) -// - fetchHook: tester callback to notify of new tasks being initiated (allows testing the scheduling logic) -// - fetch: network callback to actually send a particular download request to a physical remote peer -// - cancel: task callback to abort an in-flight download request and allow rescheduling it (in case of lost peer) -// - capacity: network callback to retrieve the estimated type-specific bandwidth capacity of a peer (traffic shaping) -// - idle: network callback to retrieve the currently (type specific) idle peers that can be assigned tasks -// - setIdle: network callback to set a peer back to idle and update its estimated capacity (traffic shaping) -// - kind: textual label of the type being downloaded to display in log messages +// - errCancel: error type to return if the fetch operation is cancelled (mostly makes logging nicer) +// - deliveryCh: channel from which to retrieve downloaded data packets (merged from all concurrent peers) +// - deliver: processing callback to deliver data packets into type specific download queues (usually within `queue`) +// - wakeCh: notification channel for waking the fetcher when new tasks are available (or sync completed) +// - expire: task callback method to abort requests that took too long and return the faulty peers (traffic shaping) +// - pending: task callback for the number of requests still needing download (detect completion/non-completability) +// - inFlight: task callback for the number of in-progress requests (wait for all active downloads to finish) +// - throttle: task callback to check if the processing queue is full and activate throttling (bound memory use) +// - reserve: task callback to reserve new download tasks to a particular peer (also signals partial completions) +// - fetchHook: tester callback to notify of new tasks being initiated (allows testing the scheduling logic) +// - fetch: network callback to actually send a particular download request to a physical remote peer +// - cancel: task callback to abort an in-flight download request and allow rescheduling it (in case of lost peer) +// - capacity: network callback to retrieve the estimated type-specific bandwidth capacity of a peer (traffic shaping) +// - idle: network callback to retrieve the currently (type specific) idle peers that can be assigned tasks +// - setIdle: network callback to set a peer back to idle and update its estimated capacity (traffic shaping) +// - kind: textual label of the type being downloaded to display in log messages func (d *Downloader) fetchParts(deliveryCh chan dataPack, deliver func(dataPack) (int, error), wakeCh chan bool, expire func() map[string]int, pending func() int, inFlight func() bool, reserve func(*peerConnection, int) (*fetchRequest, bool, bool), fetchHook func([]*types.Header), fetch func(*peerConnection, *fetchRequest) error, cancel func(*fetchRequest), capacity func(*peerConnection) int, idle func() ([]*peerConnection, int), setIdle func(*peerConnection, int, time.Time), kind string) error { - // Create a ticker to detect expired retrieval tasks ticker := time.NewTicker(100 * time.Millisecond) defer ticker.Stop() @@ -1626,7 +1627,7 @@ func (d *Downloader) processHeaders(origin uint64, td *big.Int) error { log.Warn("Invalid header encountered", "number", chunk[n].Number, "hash", chunk[n].Hash(), "parent", chunk[n].ParentHash, "err", err) return fmt.Errorf("%w: %v", errInvalidChain, err) } - // All verifications passed, track all headers within the alloted limits + // All verifications passed, track all headers within the allotted limits if mode == FastSync { head := chunk[len(chunk)-1].Number.Uint64() if head-rollback > uint64(fsHeaderSafetyNet) { @@ -1664,7 +1665,7 @@ func (d *Downloader) processHeaders(origin uint64, td *big.Int) error { } d.syncStatsLock.Unlock() - // Signal the content downloaders of the availablility of new tasks + // Signal the content downloaders of the availability of new tasks for _, ch := range []chan bool{d.bodyWakeCh, d.receiptWakeCh} { select { case ch <- true: diff --git a/les/downloader/downloader_test.go b/les/downloader/downloader_test.go index f6510eb412378..1704d3e7433a7 100644 --- a/les/downloader/downloader_test.go +++ b/les/downloader/downloader_test.go @@ -229,7 +229,7 @@ func (dl *downloadTester) CurrentFastBlock() *types.Block { func (dl *downloadTester) FastSyncCommitHead(hash common.Hash) error { // For now only check that the state trie is correct if block := dl.GetBlockByHash(hash); block != nil { - _, err := trie.NewSecure(block.Root(), trie.NewDatabase(dl.stateDb)) + _, err := trie.NewStateTrie(trie.StateTrieID(block.Root()), trie.NewDatabase(dl.stateDb)) return err } return fmt.Errorf("non existent block: %x", hash[:4]) @@ -621,7 +621,6 @@ func testThrottling(t *testing.T, protocol uint, mode SyncMode) { t.Fatalf("block synchronization failed: %v", err) } tester.terminate() - } // Tests that simple synchronization against a forked chain works correctly. In @@ -654,8 +653,8 @@ func testForkedSync(t *testing.T, protocol uint, mode SyncMode) { assertOwnForkedChain(t, tester, testChainBase.len(), []int{chainA.len(), chainB.len()}) } -// Tests that synchronising against a much shorter but much heavyer fork works -// corrently and is not dropped. +// Tests that synchronising against a much shorter but much heavier fork works +// correctly and is not dropped. func TestHeavyForkedSync66Full(t *testing.T) { testHeavyForkedSync(t, eth.ETH66, FullSync) } func TestHeavyForkedSync66Fast(t *testing.T) { testHeavyForkedSync(t, eth.ETH66, FastSync) } func TestHeavyForkedSync66Light(t *testing.T) { testHeavyForkedSync(t, eth.ETH66, LightSync) } diff --git a/les/downloader/peer.go b/les/downloader/peer.go index 8632948329711..c2161e2dae425 100644 --- a/les/downloader/peer.go +++ b/les/downloader/peer.go @@ -350,6 +350,7 @@ func (ps *peerSet) Register(p *peerConnection) error { } p.rates = msgrate.NewTracker(ps.rates.MeanCapacities(), ps.rates.MedianRoundTrip()) if err := ps.rates.Track(p.id, p.rates); err != nil { + ps.lock.Unlock() return err } ps.peers[p.id] = p @@ -413,7 +414,7 @@ func (ps *peerSet) HeaderIdlePeers() ([]*peerConnection, int) { throughput := func(p *peerConnection) int { return p.rates.Capacity(eth.BlockHeadersMsg, time.Second) } - return ps.idlePeers(eth.ETH66, eth.ETH66, idle, throughput) + return ps.idlePeers(eth.ETH66, eth.ETH67, idle, throughput) } // BodyIdlePeers retrieves a flat list of all the currently body-idle peers within @@ -425,7 +426,7 @@ func (ps *peerSet) BodyIdlePeers() ([]*peerConnection, int) { throughput := func(p *peerConnection) int { return p.rates.Capacity(eth.BlockBodiesMsg, time.Second) } - return ps.idlePeers(eth.ETH66, eth.ETH66, idle, throughput) + return ps.idlePeers(eth.ETH66, eth.ETH67, idle, throughput) } // ReceiptIdlePeers retrieves a flat list of all the currently receipt-idle peers @@ -437,7 +438,7 @@ func (ps *peerSet) ReceiptIdlePeers() ([]*peerConnection, int) { throughput := func(p *peerConnection) int { return p.rates.Capacity(eth.ReceiptsMsg, time.Second) } - return ps.idlePeers(eth.ETH66, eth.ETH66, idle, throughput) + return ps.idlePeers(eth.ETH66, eth.ETH67, idle, throughput) } // NodeDataIdlePeers retrieves a flat list of all the currently node-data-idle @@ -449,7 +450,7 @@ func (ps *peerSet) NodeDataIdlePeers() ([]*peerConnection, int) { throughput := func(p *peerConnection) int { return p.rates.Capacity(eth.NodeDataMsg, time.Second) } - return ps.idlePeers(eth.ETH66, eth.ETH66, idle, throughput) + return ps.idlePeers(eth.ETH66, eth.ETH67, idle, throughput) } // idlePeers retrieves a flat list of all currently idle peers satisfying the diff --git a/les/downloader/queue.go b/les/downloader/queue.go index 04ec12cfa9e7c..fe08c810a11f9 100644 --- a/les/downloader/queue.go +++ b/les/downloader/queue.go @@ -477,9 +477,10 @@ func (q *queue) ReserveReceipts(p *peerConnection, count int) (*fetchRequest, bo // to access the queue, so they already need a lock anyway. // // Returns: -// item - the fetchRequest -// progress - whether any progress was made -// throttle - if the caller should throttle for a while +// +// item - the fetchRequest +// progress - whether any progress was made +// throttle - if the caller should throttle for a while func (q *queue) reserveHeaders(p *peerConnection, count int, taskPool map[common.Hash]*types.Header, taskQueue *prque.Prque, pendPool map[string]*fetchRequest, kind uint) (*fetchRequest, bool, bool) { // Short circuit if the pool has been depleted, or if the peer's already @@ -833,7 +834,6 @@ func (q *queue) deliver(id string, taskPool map[common.Hash]*types.Header, taskQueue *prque.Prque, pendPool map[string]*fetchRequest, reqTimer metrics.Timer, results int, validate func(index int, header *types.Header) error, reconstruct func(index int, result *fetchResult)) (int, error) { - // Short circuit if the data was never requested request := pendPool[id] if request == nil { @@ -870,10 +870,10 @@ func (q *queue) deliver(id string, taskPool map[common.Hash]*types.Header, } for _, header := range request.Headers[:i] { - if res, stale, err := q.resultCache.GetDeliverySlot(header.Number.Uint64()); err == nil { + if res, stale, err := q.resultCache.GetDeliverySlot(header.Number.Uint64()); err == nil && !stale { reconstruct(accepted, res) } else { - // else: betweeen here and above, some other peer filled this result, + // else: between here and above, some other peer filled this result, // or it was indeed a no-op. This should not happen, but if it does it's // not something to panic about log.Error("Delivery stale", "stale", stale, "number", header.Number.Uint64(), "err", err) diff --git a/les/downloader/queue_test.go b/les/downloader/queue_test.go index 2a884d30aabac..44b2208595ff4 100644 --- a/les/downloader/queue_test.go +++ b/les/downloader/queue_test.go @@ -27,23 +27,17 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" ) -var ( - testdb = rawdb.NewMemoryDatabase() - genesis = core.GenesisBlockForTesting(testdb, testAddress, big.NewInt(1000000000000000)) -) - // makeChain creates a chain of n blocks starting at and including parent. // the returned hash chain is ordered head->parent. In addition, every 3rd block // contains a transaction and every 5th an uncle to allow testing correct block // reassembly. func makeChain(n int, seed byte, parent *types.Block, empty bool) ([]*types.Block, []types.Receipts) { - blocks, receipts := core.GenerateChain(params.TestChainConfig, parent, ethash.NewFaker(), testdb, n, func(i int, block *core.BlockGen) { + blocks, receipts := core.GenerateChain(params.TestChainConfig, parent, ethash.NewFaker(), testDB, n, func(i int, block *core.BlockGen) { block.SetCoinbase(common.Address{seed}) // Add one tx to every secondblock if !empty && i%2 == 0 { @@ -69,10 +63,10 @@ var emptyChain *chainData func init() { // Create a chain of blocks to import targetBlocks := 128 - blocks, _ := makeChain(targetBlocks, 0, genesis, false) + blocks, _ := makeChain(targetBlocks, 0, testGenesis, false) chain = &chainData{blocks, 0} - blocks, _ = makeChain(targetBlocks, 0, genesis, true) + blocks, _ = makeChain(targetBlocks, 0, testGenesis, true) emptyChain = &chainData{blocks, 0} } @@ -150,7 +144,7 @@ func TestBasics(t *testing.T) { // The second peer should hit throttling if !throttle { - t.Fatalf("should not throttle") + t.Fatalf("should throttle") } // And not get any fetches at all, since it was throttled to begin with if fetchReq != nil { @@ -179,7 +173,6 @@ func TestBasics(t *testing.T) { if got, exp := fetchReq.Headers[0].Number.Uint64(), uint64(1); got != exp { t.Fatalf("expected header %d, got %d", exp, got) } - } if exp, got := q.blockTaskQueue.Size(), numOfBlocks-10; exp != got { t.Errorf("expected block task queue to be %d, got %d", exp, got) @@ -227,7 +220,6 @@ func TestEmptyBlocks(t *testing.T) { if fetchReq != nil { t.Fatal("there should be no body fetch tasks remaining") } - } if q.blockTaskQueue.Size() != numOfBlocks-10 { t.Errorf("expected block task queue to be %d, got %d", numOfBlocks-10, q.blockTaskQueue.Size()) @@ -241,7 +233,7 @@ func TestEmptyBlocks(t *testing.T) { // there should be nothing to fetch, blocks are empty if fetchReq != nil { - t.Fatal("there should be no body fetch tasks remaining") + t.Fatal("there should be no receipt fetch tasks remaining") } } if q.blockTaskQueue.Size() != numOfBlocks-10 { @@ -261,14 +253,13 @@ func TestEmptyBlocks(t *testing.T) { // some more advanced scenarios func XTestDelivery(t *testing.T) { // the outside network, holding blocks - blo, rec := makeChain(128, 0, genesis, false) + blo, rec := makeChain(128, 0, testGenesis, false) world := newNetwork() world.receipts = rec world.chain = blo world.progress(10) if false { log.Root().SetHandler(log.StdoutHandler) - } q := newQueue(10, 10) var wg sync.WaitGroup @@ -299,7 +290,6 @@ func XTestDelivery(t *testing.T) { fmt.Printf("got %d results, %d tot\n", len(res), tot) // Now we can forget about these world.forget(res[len(res)-1].Header.Number.Uint64()) - } }() wg.Add(1) @@ -362,7 +352,6 @@ func XTestDelivery(t *testing.T) { } for i := 0; i < 50; i++ { time.Sleep(2990 * time.Millisecond) - } }() wg.Add(1) @@ -413,10 +402,8 @@ func (n *network) forget(blocknum uint64) { n.chain = n.chain[index:] n.receipts = n.receipts[index:] n.offset = int(blocknum) - } func (n *network) progress(numBlocks int) { - n.lock.Lock() defer n.lock.Unlock() //fmt.Printf("progressing...\n") @@ -424,7 +411,6 @@ func (n *network) progress(numBlocks int) { n.chain = append(n.chain, newBlocks...) n.receipts = append(n.receipts, newR...) n.cond.Broadcast() - } func (n *network) headers(from int) []*types.Header { diff --git a/les/downloader/resultstore.go b/les/downloader/resultstore.go index 3162cd6d5b42e..2dcbbe16c916c 100644 --- a/les/downloader/resultstore.go +++ b/les/downloader/resultstore.go @@ -71,10 +71,11 @@ func (r *resultStore) SetThrottleThreshold(threshold uint64) uint64 { // wants to reserve headers for fetching. // // It returns the following: -// stale - if true, this item is already passed, and should not be requested again -// throttled - if true, the store is at capacity, this particular header is not prio now -// item - the result to store data into -// err - any error that occurred +// +// stale - if true, this item is already passed, and should not be requested again +// throttled - if true, the store is at capacity, this particular header is not prio now +// item - the result to store data into +// err - any error that occurred func (r *resultStore) AddFetch(header *types.Header, fastSync bool) (stale, throttled bool, item *fetchResult, err error) { r.lock.Lock() defer r.lock.Unlock() diff --git a/les/downloader/statesync.go b/les/downloader/statesync.go index 2b3278822996b..22f952155f119 100644 --- a/les/downloader/statesync.go +++ b/les/downloader/statesync.go @@ -34,7 +34,7 @@ import ( // a single data retrieval network packet. type stateReq struct { nItems uint16 // Number of items requested for download (max is 384, so uint16 is sufficient) - trieTasks map[common.Hash]*trieTask // Trie node download tasks to track previous attempts + trieTasks map[string]*trieTask // Trie node download tasks to track previous attempts codeTasks map[common.Hash]*codeTask // Byte code download tasks to track previous attempts timeout time.Duration // Maximum round trip time for this to complete timer *time.Timer // Timer to fire when the RTT timeout expires @@ -263,8 +263,8 @@ type stateSync struct { sched *trie.Sync // State trie sync scheduler defining the tasks keccak crypto.KeccakState // Keccak256 hasher to verify deliveries with - trieTasks map[common.Hash]*trieTask // Set of trie node tasks currently queued for retrieval - codeTasks map[common.Hash]*codeTask // Set of byte code tasks currently queued for retrieval + trieTasks map[string]*trieTask // Set of trie node tasks currently queued for retrieval, indexed by path + codeTasks map[common.Hash]*codeTask // Set of byte code tasks currently queued for retrieval, indexed by hash numUncommitted int bytesUncommitted int @@ -281,6 +281,7 @@ type stateSync struct { // trieTask represents a single trie node download task, containing a set of // peers already attempted retrieval from to detect stalled syncs and abort. type trieTask struct { + hash common.Hash path [][]byte attempts map[string]struct{} } @@ -299,7 +300,7 @@ func newStateSync(d *Downloader, root common.Hash) *stateSync { root: root, sched: state.NewStateSync(root, d.stateDB, nil), keccak: sha3.NewLegacyKeccak256().(crypto.KeccakState), - trieTasks: make(map[common.Hash]*trieTask), + trieTasks: make(map[string]*trieTask), codeTasks: make(map[common.Hash]*codeTask), deliver: make(chan *stateReq), cancel: make(chan struct{}), @@ -455,10 +456,11 @@ func (s *stateSync) assignTasks() { func (s *stateSync) fillTasks(n int, req *stateReq) (nodes []common.Hash, paths []trie.SyncPath, codes []common.Hash) { // Refill available tasks from the scheduler. if fill := n - (len(s.trieTasks) + len(s.codeTasks)); fill > 0 { - nodes, paths, codes := s.sched.Missing(fill) - for i, hash := range nodes { - s.trieTasks[hash] = &trieTask{ - path: paths[i], + paths, hashes, codes := s.sched.Missing(fill) + for i, path := range paths { + s.trieTasks[path] = &trieTask{ + hash: hashes[i], + path: trie.NewSyncPath([]byte(path)), attempts: make(map[string]struct{}), } } @@ -474,7 +476,7 @@ func (s *stateSync) fillTasks(n int, req *stateReq) (nodes []common.Hash, paths paths = make([]trie.SyncPath, 0, n) codes = make([]common.Hash, 0, n) - req.trieTasks = make(map[common.Hash]*trieTask, n) + req.trieTasks = make(map[string]*trieTask, n) req.codeTasks = make(map[common.Hash]*codeTask, n) for hash, t := range s.codeTasks { @@ -492,7 +494,7 @@ func (s *stateSync) fillTasks(n int, req *stateReq) (nodes []common.Hash, paths req.codeTasks[hash] = t delete(s.codeTasks, hash) } - for hash, t := range s.trieTasks { + for path, t := range s.trieTasks { // Stop when we've gathered enough requests if len(nodes)+len(codes) == n { break @@ -504,11 +506,11 @@ func (s *stateSync) fillTasks(n int, req *stateReq) (nodes []common.Hash, paths // Assign the request to this peer t.attempts[req.peer.id] = struct{}{} - nodes = append(nodes, hash) + nodes = append(nodes, t.hash) paths = append(paths, t.path) - req.trieTasks[hash] = t - delete(s.trieTasks, hash) + req.trieTasks[path] = t + delete(s.trieTasks, path) } req.nItems = uint16(len(nodes) + len(codes)) return nodes, paths, codes @@ -530,7 +532,7 @@ func (s *stateSync) process(req *stateReq) (int, error) { // Iterate over all the delivered data and inject one-by-one into the trie for _, blob := range req.response { - hash, err := s.processNodeData(blob) + hash, err := s.processNodeData(req.trieTasks, req.codeTasks, blob) switch err { case nil: s.numUncommitted++ @@ -543,13 +545,10 @@ func (s *stateSync) process(req *stateReq) (int, error) { default: return successful, fmt.Errorf("invalid state node %s: %v", hash.TerminalString(), err) } - // Delete from both queues (one delivery is enough for the syncer) - delete(req.trieTasks, hash) - delete(req.codeTasks, hash) } // Put unfulfilled tasks back into the retry queue npeers := s.d.peers.Len() - for hash, task := range req.trieTasks { + for path, task := range req.trieTasks { // If the node did deliver something, missing items may be due to a protocol // limit or a previous timeout + delayed delivery. Both cases should permit // the node to retry the missing items (to avoid single-peer stalls). @@ -559,10 +558,10 @@ func (s *stateSync) process(req *stateReq) (int, error) { // If we've requested the node too many times already, it may be a malicious // sync where nobody has the right data. Abort. if len(task.attempts) >= npeers { - return successful, fmt.Errorf("trie node %s failed with all peers (%d tries, %d peers)", hash.TerminalString(), len(task.attempts), npeers) + return successful, fmt.Errorf("trie node %s failed with all peers (%d tries, %d peers)", task.hash.TerminalString(), len(task.attempts), npeers) } // Missing item, place into the retry queue. - s.trieTasks[hash] = task + s.trieTasks[path] = task } for hash, task := range req.codeTasks { // If the node did deliver something, missing items may be due to a protocol @@ -585,13 +584,35 @@ func (s *stateSync) process(req *stateReq) (int, error) { // processNodeData tries to inject a trie node data blob delivered from a remote // peer into the state trie, returning whether anything useful was written or any // error occurred. -func (s *stateSync) processNodeData(blob []byte) (common.Hash, error) { - res := trie.SyncResult{Data: blob} +// +// If multiple requests correspond to the same hash, this method will inject the +// blob as a result for the first one only, leaving the remaining duplicates to +// be fetched again. +func (s *stateSync) processNodeData(nodeTasks map[string]*trieTask, codeTasks map[common.Hash]*codeTask, blob []byte) (common.Hash, error) { + var hash common.Hash s.keccak.Reset() s.keccak.Write(blob) - s.keccak.Read(res.Hash[:]) - err := s.sched.Process(res) - return res.Hash, err + s.keccak.Read(hash[:]) + + if _, present := codeTasks[hash]; present { + err := s.sched.ProcessCode(trie.CodeSyncResult{ + Hash: hash, + Data: blob, + }) + delete(codeTasks, hash) + return hash, err + } + for path, task := range nodeTasks { + if task.hash == hash { + err := s.sched.ProcessNode(trie.NodeSyncResult{ + Path: path, + Data: blob, + }) + delete(nodeTasks, path) + return hash, err + } + } + return common.Hash{}, trie.ErrNotRequested } // updateStats bumps the various state sync progress counters and displays a log @@ -608,7 +629,7 @@ func (s *stateSync) updateStats(written, duplicate, unexpected int, duration tim if written > 0 || duplicate > 0 || unexpected > 0 { log.Info("Imported new state entries", "count", written, "elapsed", common.PrettyDuration(duration), "processed", s.d.syncStatsState.processed, "pending", s.d.syncStatsState.pending, "trieretry", len(s.trieTasks), "coderetry", len(s.codeTasks), "duplicate", s.d.syncStatsState.duplicate, "unexpected", s.d.syncStatsState.unexpected) } - if written > 0 { - //rawdb.WriteFastTrieProgress(s.d.stateDB, s.d.syncStatsState.processed) - } + //if written > 0 { + //rawdb.WriteFastTrieProgress(s.d.stateDB, s.d.syncStatsState.processed) + //} } diff --git a/les/downloader/testchain_test.go b/les/downloader/testchain_test.go index b9865f7e032b3..400eec94e7c49 100644 --- a/les/downloader/testchain_test.go +++ b/les/downloader/testchain_test.go @@ -35,7 +35,12 @@ var ( testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") testAddress = crypto.PubkeyToAddress(testKey.PublicKey) testDB = rawdb.NewMemoryDatabase() - testGenesis = core.GenesisBlockForTesting(testDB, testAddress, big.NewInt(1000000000000000)) + + gspec = core.Genesis{ + Alloc: core.GenesisAlloc{testAddress: {Balance: big.NewInt(1000000000000000)}}, + BaseFee: big.NewInt(params.InitialBaseFee), + } + testGenesis = gspec.MustCommit(testDB) ) // The common prefix of all test chains: diff --git a/les/fetcher.go b/les/fetcher.go index cf62c8f707765..ef37d80cd6f44 100644 --- a/les/fetcher.go +++ b/les/fetcher.go @@ -242,18 +242,20 @@ func (f *lightFetcher) forEachPeer(check func(id enode.ID, p *fetcherPeer) bool) } // mainloop is the main event loop of the light fetcher, which is responsible for -// - announcement maintenance(ulc) -// If we are running in ultra light client mode, then all announcements from -// the trusted servers are maintained. If the same announcements from trusted -// servers reach the threshold, then the relevant header is requested for retrieval. // -// - block header retrieval -// Whenever we receive announce with higher td compared with local chain, the -// request will be made for header retrieval. +// - announcement maintenance(ulc) // -// - re-sync trigger -// If the local chain lags too much, then the fetcher will enter "synnchronise" -// mode to retrieve missing headers in batch. +// If we are running in ultra light client mode, then all announcements from +// the trusted servers are maintained. If the same announcements from trusted +// servers reach the threshold, then the relevant header is requested for retrieval. +// +// - block header retrieval +// Whenever we receive announce with higher td compared with local chain, the +// request will be made for header retrieval. +// +// - re-sync trigger +// If the local chain lags too much, then the fetcher will enter "synchronise" +// mode to retrieve missing headers in batch. func (f *lightFetcher) mainloop() { defer f.wg.Done() diff --git a/les/fetcher/block_fetcher.go b/les/fetcher/block_fetcher.go index 283008db0f1e5..86b3c552ce27d 100644 --- a/les/fetcher/block_fetcher.go +++ b/les/fetcher/block_fetcher.go @@ -641,7 +641,6 @@ func (f *BlockFetcher) loop() { } else { f.forgetHash(hash) } - } if matched { task.transactions = append(task.transactions[:i], task.transactions[i+1:]...) diff --git a/les/fetcher/block_fetcher_test.go b/les/fetcher/block_fetcher_test.go index de066ac26b5ff..caff7a3b3559a 100644 --- a/les/fetcher/block_fetcher_test.go +++ b/les/fetcher/block_fetcher_test.go @@ -35,10 +35,15 @@ import ( ) var ( - testdb = rawdb.NewMemoryDatabase() - testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - testAddress = crypto.PubkeyToAddress(testKey.PublicKey) - genesis = core.GenesisBlockForTesting(testdb, testAddress, big.NewInt(1000000000000000)) + testdb = rawdb.NewMemoryDatabase() + testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + testAddress = crypto.PubkeyToAddress(testKey.PublicKey) + + gspec = core.Genesis{ + Alloc: core.GenesisAlloc{testAddress: {Balance: big.NewInt(1000000000000000)}}, + BaseFee: big.NewInt(params.InitialBaseFee), + } + genesis = gspec.MustCommit(testdb) unknownBlock = types.NewBlock(&types.Header{GasLimit: params.GenesisGasLimit, BaseFee: big.NewInt(params.InitialBaseFee)}, nil, nil, nil, trie.NewStackTrie(nil)) ) diff --git a/les/fetcher_test.go b/les/fetcher_test.go index 28db3b8913aca..6a17e73757a54 100644 --- a/les/fetcher_test.go +++ b/les/fetcher_test.go @@ -160,7 +160,6 @@ func testTrustedAnnouncement(t *testing.T, protocol int) { nodes []*enode.Node ids []string cpeers []*clientPeer - speers []*serverPeer config = light.TestServerIndexerConfig waitIndexers = func(cIndexer, bIndexer, btIndexer *core.ChainIndexer) { @@ -213,12 +212,11 @@ func testTrustedAnnouncement(t *testing.T, protocol int) { // Connect all server instances. for i := 0; i < len(servers); i++ { - sp, cp, err := connect(servers[i].handler, nodes[i].ID(), c.handler, protocol, true) + _, cp, err := connect(servers[i].handler, nodes[i].ID(), c.handler, protocol, true) if err != nil { t.Fatalf("connect server and client failed, err %s", err) } cpeers = append(cpeers, cp) - speers = append(speers, sp) } newHead := make(chan *types.Header, 1) c.handler.fetcher.newHeadHook = func(header *types.Header) { newHead <- header } diff --git a/les/flowcontrol/control.go b/les/flowcontrol/control.go index 4f0de82318357..76a241fa5a7f0 100644 --- a/les/flowcontrol/control.go +++ b/les/flowcontrol/control.go @@ -182,7 +182,7 @@ func (node *ClientNode) UpdateParams(params ServerParams) { return } } - node.updateSchedule = append(node.updateSchedule, scheduledUpdate{time: now + mclock.AbsTime(DecParamDelay), params: params}) + node.updateSchedule = append(node.updateSchedule, scheduledUpdate{time: now.Add(DecParamDelay), params: params}) } } diff --git a/les/flowcontrol/manager.go b/les/flowcontrol/manager.go index c9e681c1440a8..4367974d632e2 100644 --- a/les/flowcontrol/manager.go +++ b/les/flowcontrol/manager.go @@ -55,13 +55,12 @@ var ( // ClientManager controls the capacity assigned to the clients of a server. // Since ServerParams guarantee a safe lower estimate for processable requests // even in case of all clients being active, ClientManager calculates a -// corrigated buffer value and usually allows a higher remaining buffer value +// corrugated buffer value and usually allows a higher remaining buffer value // to be returned with each reply. type ClientManager struct { - clock mclock.Clock - lock sync.Mutex - enabledCh chan struct{} - stop chan chan struct{} + clock mclock.Clock + lock sync.Mutex + stop chan chan struct{} curve PieceWiseLinear sumRecharge, totalRecharge, totalConnected uint64 diff --git a/les/flowcontrol/manager_test.go b/les/flowcontrol/manager_test.go index 9d2f88763614a..564d813f15a38 100644 --- a/les/flowcontrol/manager_test.go +++ b/les/flowcontrol/manager_test.go @@ -104,7 +104,6 @@ func testConstantTotalCapacity(t *testing.T, nodeCount, maxCapacityNodes, random if ratio < 0.98 || ratio > 1.02 { t.Errorf("totalCost/totalCapacity/testLength ratio incorrect (expected: 1, got: %f)", ratio) } - } func (n *testNode) send(t *testing.T, now mclock.AbsTime) bool { diff --git a/les/handler_test.go b/les/handler_test.go index aba45764b3068..ecf97bf9d1097 100644 --- a/les/handler_test.go +++ b/les/handler_test.go @@ -405,7 +405,7 @@ func testGetProofs(t *testing.T, protocol int) { accounts := []common.Address{bankAddr, userAddr1, userAddr2, signerAddr, {}} for i := uint64(0); i <= bc.CurrentBlock().NumberU64(); i++ { header := bc.GetHeaderByNumber(i) - trie, _ := trie.New(header.Root, trie.NewDatabase(server.db)) + trie, _ := trie.New(trie.StateTrieID(header.Root), trie.NewDatabase(server.db)) for _, acc := range accounts { req := ProofReq{ @@ -456,7 +456,7 @@ func testGetStaleProof(t *testing.T, protocol int) { var expected []rlp.RawValue if wantOK { proofsV2 := light.NewNodeSet() - t, _ := trie.New(header.Root, trie.NewDatabase(server.db)) + t, _ := trie.New(trie.StateTrieID(header.Root), trie.NewDatabase(server.db)) t.Prove(account, 0, proofsV2) expected = proofsV2.NodeList() } @@ -512,7 +512,7 @@ func testGetCHTProofs(t *testing.T, protocol int) { AuxData: [][]byte{rlp}, } root := light.GetChtRoot(server.db, 0, bc.GetHeaderByNumber(config.ChtSize-1).Hash()) - trie, _ := trie.New(root, trie.NewDatabase(rawdb.NewTable(server.db, light.ChtTablePrefix))) + trie, _ := trie.New(trie.TrieID(root), trie.NewDatabase(rawdb.NewTable(server.db, light.ChtTablePrefix))) trie.Prove(key, 0, &proofsV2.Proofs) // Assemble the requests for the different protocols requestsV2 := []HelperTrieReq{{ @@ -577,7 +577,7 @@ func testGetBloombitsProofs(t *testing.T, protocol int) { var proofs HelperTrieResps root := light.GetBloomTrieRoot(server.db, 0, bc.GetHeaderByNumber(config.BloomTrieSize-1).Hash()) - trie, _ := trie.New(root, trie.NewDatabase(rawdb.NewTable(server.db, light.BloomTrieTablePrefix))) + trie, _ := trie.New(trie.TrieID(root), trie.NewDatabase(rawdb.NewTable(server.db, light.BloomTrieTablePrefix))) trie.Prove(key, 0, &proofs.Proofs) // Send the proof request and verify the response diff --git a/les/odr.go b/les/odr.go index 10ff0854d3850..2643a534787f5 100644 --- a/les/odr.go +++ b/les/odr.go @@ -126,7 +126,7 @@ const ( // RetrieveTxStatus retrieves the transaction status from the LES network. // There is no guarantee in the LES protocol that the mined transaction will // be retrieved back for sure because of different reasons(the transaction -// is unindexed, the malicous server doesn't reply it deliberately, etc). +// is unindexed, the malicious server doesn't reply it deliberately, etc). // Therefore, unretrieved transactions(UNKNOWN) will receive a certain number // of retries, thus giving a weak guarantee. func (odr *LesOdr) RetrieveTxStatus(ctx context.Context, req *light.TxStatusRequest) error { diff --git a/les/odr_test.go b/les/odr_test.go index ad77abf5b9b28..48fd9f95e394f 100644 --- a/les/odr_test.go +++ b/les/odr_test.go @@ -129,7 +129,7 @@ func odrContractCall(ctx context.Context, db ethdb.Database, config *params.Chai data[35] = byte(i) if bc != nil { header := bc.GetHeaderByHash(bhash) - statedb, err := state.New(header.Root, state.NewDatabase(db), nil) + statedb, err := state.New(header.Root, bc.StateCache(), nil) if err == nil { from := statedb.GetOrNewStateObject(bankAddr) @@ -392,12 +392,10 @@ func testGetTxStatusFromUnindexedPeers(t *testing.T, protocol int) { for _, testspec := range testspecs { // Create a bunch of server peers with different tx history var ( - serverPeers []*testPeer - closeFns []func() + closeFns []func() ) for i := 0; i < testspec.peers; i++ { peer, closePeer, _ := client.newRawPeer(t, fmt.Sprintf("server-%d", i), protocol, testspec.txLookups[i]) - serverPeers = append(serverPeers, peer) closeFns = append(closeFns, closePeer) // Create a one-time routine for serving message diff --git a/les/peer.go b/les/peer.go index 499429739d23d..deda052a3b14b 100644 --- a/les/peer.go +++ b/les/peer.go @@ -995,40 +995,6 @@ func (p *clientPeer) sendLastAnnounce() { } } -// freezeClient temporarily puts the client in a frozen state which means all -// unprocessed and subsequent requests are dropped. Unfreezing happens automatically -// after a short time if the client's buffer value is at least in the slightly positive -// region. The client is also notified about being frozen/unfrozen with a Stop/Resume -// message. -func (p *clientPeer) freezeClient() { - if p.version < lpv3 { - // if Stop/Resume is not supported then just drop the peer after setting - // its frozen status permanently - atomic.StoreUint32(&p.frozen, 1) - p.Peer.Disconnect(p2p.DiscUselessPeer) - return - } - if atomic.SwapUint32(&p.frozen, 1) == 0 { - go func() { - p.sendStop() - time.Sleep(freezeTimeBase + time.Duration(rand.Int63n(int64(freezeTimeRandom)))) - for { - bufValue, bufLimit := p.fcClient.BufferStatus() - if bufLimit == 0 { - return - } - if bufValue <= bufLimit/8 { - time.Sleep(freezeCheckPeriod) - } else { - atomic.StoreUint32(&p.frozen, 0) - p.sendResume(bufValue) - break - } - } - }() - } -} - // Handshake executes the les protocol handshake, negotiating version number, // network IDs, difficulties, head and genesis blocks. func (p *clientPeer) Handshake(td *big.Int, head common.Hash, headNum uint64, genesis common.Hash, forkID forkid.ID, forkFilter forkid.Filter, server *LesServer) error { @@ -1157,19 +1123,6 @@ func (ps *serverPeerSet) subscribe(sub serverPeerSubscriber) { } } -// unSubscribe removes the specified service from the subscriber pool. -func (ps *serverPeerSet) unSubscribe(sub serverPeerSubscriber) { - ps.lock.Lock() - defer ps.lock.Unlock() - - for i, s := range ps.subscribers { - if s == sub { - ps.subscribers = append(ps.subscribers[:i], ps.subscribers[i+1:]...) - return - } - } -} - // register adds a new server peer into the set, or returns an error if the // peer is already known. func (ps *serverPeerSet) register(peer *serverPeer) error { @@ -1236,25 +1189,6 @@ func (ps *serverPeerSet) len() int { return len(ps.peers) } -// bestPeer retrieves the known peer with the currently highest total difficulty. -// If the peerset is "client peer set", then nothing meaningful will return. The -// reason is client peer never send back their latest status to server. -func (ps *serverPeerSet) bestPeer() *serverPeer { - ps.lock.RLock() - defer ps.lock.RUnlock() - - var ( - bestPeer *serverPeer - bestTd *big.Int - ) - for _, p := range ps.peers { - if td := p.Td(); bestTd == nil || td.Cmp(bestTd) > 0 { - bestPeer, bestTd = p, td - } - } - return bestPeer -} - // allServerPeers returns all server peers in a list. func (ps *serverPeerSet) allPeers() []*serverPeer { ps.lock.RLock() @@ -1348,14 +1282,6 @@ func (ps *clientPeerSet) peer(id enode.ID) *clientPeer { return ps.peers[id] } -// len returns if the current number of peers in the set. -func (ps *clientPeerSet) len() int { - ps.lock.RLock() - defer ps.lock.RUnlock() - - return len(ps.peers) -} - // setSignerKey sets the signer key for signed announcements. Should be called before // starting the protocol handler. func (ps *clientPeerSet) setSignerKey(privateKey *ecdsa.PrivateKey) { diff --git a/les/peer_test.go b/les/peer_test.go index d6551ce6b6393..b8a1482a040a9 100644 --- a/les/peer_test.go +++ b/les/peer_test.go @@ -28,7 +28,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/forkid" - "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" @@ -100,7 +99,7 @@ type fakeChain struct{} func (f *fakeChain) Config() *params.ChainConfig { return params.MainnetChainConfig } func (f *fakeChain) Genesis() *types.Block { - return core.DefaultGenesisBlock().ToBlock(rawdb.NewMemoryDatabase()) + return core.DefaultGenesisBlock().ToBlock() } func (f *fakeChain) CurrentHeader() *types.Header { return &types.Header{Number: big.NewInt(10000000)} } diff --git a/les/request_test.go b/les/request_test.go index c65405e375228..9b52e6bd86ad8 100644 --- a/les/request_test.go +++ b/les/request_test.go @@ -104,6 +104,7 @@ func testAccess(t *testing.T, protocol int, fn accessTestFn) { bhash := rawdb.ReadCanonicalHash(server.db, i) if req := fn(client.db, bhash, i); req != nil { ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + err := client.handler.backend.odr.Retrieve(ctx, req) cancel() diff --git a/les/server.go b/les/server.go index c135e65f2dc4b..df453b4819a2e 100644 --- a/les/server.go +++ b/les/server.go @@ -159,21 +159,15 @@ func (s *LesServer) APIs() []rpc.API { return []rpc.API{ { Namespace: "les", - Version: "1.0", - Service: NewPrivateLightAPI(&s.lesCommons), - Public: false, + Service: NewLightAPI(&s.lesCommons), }, { Namespace: "les", - Version: "1.0", - Service: NewPrivateLightServerAPI(s), - Public: false, + Service: NewLightServerAPI(s), }, { Namespace: "debug", - Version: "1.0", - Service: NewPrivateDebugAPI(s), - Public: false, + Service: NewDebugAPI(s), }, } } diff --git a/les/server_handler.go b/les/server_handler.go index ef1af844c26b1..32a38f64cc44e 100644 --- a/les/server_handler.go +++ b/les/server_handler.go @@ -359,7 +359,7 @@ func (h *serverHandler) AddTxsSync() bool { // getAccount retrieves an account from the state based on root. func getAccount(triedb *trie.Database, root, hash common.Hash) (types.StateAccount, error) { - trie, err := trie.New(root, triedb) + trie, err := trie.New(trie.StateTrieID(root), triedb) if err != nil { return types.StateAccount{}, err } @@ -391,7 +391,7 @@ func (h *serverHandler) GetHelperTrie(typ uint, index uint64) *trie.Trie { if root == (common.Hash{}) { return nil } - trie, _ := trie.New(root, trie.NewDatabase(rawdb.NewTable(h.chainDb, prefix))) + trie, _ := trie.New(trie.TrieID(root), trie.NewDatabase(rawdb.NewTable(h.chainDb, prefix))) return trie } diff --git a/les/server_requests.go b/les/server_requests.go index bab5f733d549b..b0eb2371e028e 100644 --- a/les/server_requests.go +++ b/les/server_requests.go @@ -428,7 +428,7 @@ func handleGetProofs(msg Decoder) (serveRequestFn, uint64, uint64, error) { p.bumpInvalid() continue } - trie, err = statedb.OpenStorageTrie(common.BytesToHash(request.AccKey), account.Root) + trie, err = statedb.OpenStorageTrie(root, common.BytesToHash(request.AccKey), account.Root) if trie == nil || err != nil { p.Log().Warn("Failed to open storage trie for proof", "block", header.Number, "hash", header.Hash(), "account", common.BytesToHash(request.AccKey), "root", account.Root, "err", err) continue diff --git a/les/state_accessor.go b/les/state_accessor.go index 112e6fd44d12a..a2d49fbf31ce0 100644 --- a/les/state_accessor.go +++ b/les/state_accessor.go @@ -25,31 +25,36 @@ import ( "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/eth/tracers" "github.com/ethereum/go-ethereum/light" ) +// noopReleaser is returned in case there is no operation expected +// for releasing state. +var noopReleaser = tracers.StateReleaseFunc(func() {}) + // stateAtBlock retrieves the state database associated with a certain block. -func (leth *LightEthereum) stateAtBlock(ctx context.Context, block *types.Block, reexec uint64) (*state.StateDB, error) { - return light.NewState(ctx, block.Header(), leth.odr), nil +func (leth *LightEthereum) stateAtBlock(ctx context.Context, block *types.Block, reexec uint64) (*state.StateDB, tracers.StateReleaseFunc, error) { + return light.NewState(ctx, block.Header(), leth.odr), noopReleaser, nil } // stateAtTransaction returns the execution environment of a certain transaction. -func (leth *LightEthereum) stateAtTransaction(ctx context.Context, block *types.Block, txIndex int, reexec uint64) (core.Message, vm.BlockContext, *state.StateDB, error) { +func (leth *LightEthereum) stateAtTransaction(ctx context.Context, block *types.Block, txIndex int, reexec uint64) (core.Message, vm.BlockContext, *state.StateDB, tracers.StateReleaseFunc, error) { // Short circuit if it's genesis block. if block.NumberU64() == 0 { - return nil, vm.BlockContext{}, nil, errors.New("no transaction in genesis") + return nil, vm.BlockContext{}, nil, nil, errors.New("no transaction in genesis") } // Create the parent state database parent, err := leth.blockchain.GetBlock(ctx, block.ParentHash(), block.NumberU64()-1) if err != nil { - return nil, vm.BlockContext{}, nil, err + return nil, vm.BlockContext{}, nil, nil, err } - statedb, err := leth.stateAtBlock(ctx, parent, reexec) + statedb, release, err := leth.stateAtBlock(ctx, parent, reexec) if err != nil { - return nil, vm.BlockContext{}, nil, err + return nil, vm.BlockContext{}, nil, nil, err } if txIndex == 0 && len(block.Transactions()) == 0 { - return nil, vm.BlockContext{}, statedb, nil + return nil, vm.BlockContext{}, statedb, release, nil } // Recompute transactions up to the target index. signer := types.MakeSigner(leth.blockchain.Config(), block.Number()) @@ -60,16 +65,16 @@ func (leth *LightEthereum) stateAtTransaction(ctx context.Context, block *types. context := core.NewEVMBlockContext(block.Header(), leth.blockchain, nil) statedb.Prepare(tx.Hash(), idx) if idx == txIndex { - return msg, context, statedb, nil + return msg, context, statedb, release, nil } // Not yet the searched for transaction, execute on top of the current state vmenv := vm.NewEVM(context, txContext, statedb, leth.blockchain.Config(), vm.Config{}) if _, err := core.ApplyMessage(vmenv, msg, new(core.GasPool).AddGas(tx.Gas())); err != nil { - return nil, vm.BlockContext{}, nil, fmt.Errorf("transaction %#x failed: %v", tx.Hash(), err) + return nil, vm.BlockContext{}, nil, nil, fmt.Errorf("transaction %#x failed: %v", tx.Hash(), err) } // Ensure any modifications are committed to the state // Only delete empty objects if EIP158/161 (a.k.a Spurious Dragon) is in effect statedb.Finalise(vmenv.ChainConfig().IsEIP158(block.Number())) } - return nil, vm.BlockContext{}, nil, fmt.Errorf("transaction index %d out of range for block %#x", txIndex, block.Hash()) + return nil, vm.BlockContext{}, nil, nil, fmt.Errorf("transaction index %d out of range for block %#x", txIndex, block.Hash()) } diff --git a/les/ulc_test.go b/les/ulc_test.go index a4df0795b46d4..9a29a24cee55b 100644 --- a/les/ulc_test.go +++ b/les/ulc_test.go @@ -35,6 +35,19 @@ func TestULCAnnounceThresholdLes3(t *testing.T) { testULCAnnounceThreshold(t, 3) func testULCAnnounceThreshold(t *testing.T, protocol int) { // todo figure out why it takes fetcher so longer to fetcher the announced header. t.Skip("Sometimes it can failed") + + // newTestLightPeer creates node with light sync mode + newTestLightPeer := func(t *testing.T, protocol int, ulcServers []string, ulcFraction int) (*testClient, func()) { + netconfig := testnetConfig{ + protocol: protocol, + ulcServers: ulcServers, + ulcFraction: ulcFraction, + nopruning: true, + } + _, c, teardown := newClientServerEnv(t, netconfig) + return c, teardown + } + var cases = []struct { height []int threshold int @@ -148,15 +161,3 @@ func newTestServerPeer(t *testing.T, blocks int, protocol int, indexFn indexerCa n := enode.NewV4(&key.PublicKey, net.ParseIP("127.0.0.1"), 35000, 35000) return s, n, teardown } - -// newTestLightPeer creates node with light sync mode -func newTestLightPeer(t *testing.T, protocol int, ulcServers []string, ulcFraction int) (*testClient, func()) { - netconfig := testnetConfig{ - protocol: protocol, - ulcServers: ulcServers, - ulcFraction: ulcFraction, - nopruning: true, - } - _, c, teardown := newClientServerEnv(t, netconfig) - return c, teardown -} diff --git a/les/utils/timeutils_test.go b/les/utils/timeutils_test.go index 9f9e1c2dc9389..b219d0439dcb2 100644 --- a/les/utils/timeutils_test.go +++ b/les/utils/timeutils_test.go @@ -37,7 +37,7 @@ func TestUpdateTimer(t *testing.T) { if updated := timer.Update(func(diff time.Duration) bool { return true }); !updated { t.Fatalf("Doesn't update the clock when reaching the threshold") } - if updated := timer.UpdateAt(sim.Now()+mclock.AbsTime(time.Second), func(diff time.Duration) bool { return true }); !updated { + if updated := timer.UpdateAt(sim.Now().Add(time.Second), func(diff time.Duration) bool { return true }); !updated { t.Fatalf("Doesn't update the clock when reaching the threshold") } timer = NewUpdateTimer(sim, 0) diff --git a/les/vflux/client/fillset_test.go b/les/vflux/client/fillset_test.go index ca5af8f07ecc9..ddb12a82f9b3a 100644 --- a/les/vflux/client/fillset_test.go +++ b/les/vflux/client/fillset_test.go @@ -104,7 +104,7 @@ func TestFillSet(t *testing.T) { fs.SetTarget(10) expWaiting(4, true) expNotWaiting() - // remove all previosly set flags + // remove all previously set flags ns.ForEach(sfTest1, nodestate.Flags{}, func(node *enode.Node, state nodestate.Flags) { ns.SetState(node, nodestate.Flags{}, sfTest1, 0) }) diff --git a/les/vflux/client/serverpool.go b/les/vflux/client/serverpool.go index e481075f70bd8..cf96f0ee3a23b 100644 --- a/les/vflux/client/serverpool.go +++ b/les/vflux/client/serverpool.go @@ -222,7 +222,6 @@ func (s *serverPoolIterator) Close() { func (s *ServerPool) AddMetrics( suggestedTimeoutGauge, totalValueGauge, serverSelectableGauge, serverConnectedGauge metrics.Gauge, sessionValueMeter, serverDialedMeter metrics.Meter) { - s.suggestedTimeoutGauge = suggestedTimeoutGauge s.totalValueGauge = totalValueGauge s.sessionValueMeter = sessionValueMeter diff --git a/les/vflux/client/serverpool_test.go b/les/vflux/client/serverpool_test.go index c7d0245ef21d9..f1fd987d7edb1 100644 --- a/les/vflux/client/serverpool_test.go +++ b/les/vflux/client/serverpool_test.go @@ -55,7 +55,6 @@ type ServerPoolTest struct { clock *mclock.Simulated quit chan chan struct{} preNeg, preNegFail bool - vt *ValueTracker sp *ServerPool spi enode.Iterator input enode.Iterator @@ -67,7 +66,7 @@ type ServerPoolTest struct { // (accessed from both the main thread and the preNeg callback) preNegLock sync.Mutex queryWg *sync.WaitGroup // a new wait group is created each time the simulation is started - stopping bool // stopping avoid callind queryWg.Add after queryWg.Wait + stopping bool // stopping avoid calling queryWg.Add after queryWg.Wait cycle, conn, servedConn int serviceCycles, dialCount int diff --git a/les/vflux/client/wrsiterator.go b/les/vflux/client/wrsiterator.go index 8a2e39ad44228..1b37cba6e5ded 100644 --- a/les/vflux/client/wrsiterator.go +++ b/les/vflux/client/wrsiterator.go @@ -109,7 +109,6 @@ func (w *WrsIterator) chooseNode() *enode.Node { return w.ns.GetNode(id) } } - } // Close ends the iterator. diff --git a/les/vflux/server/balance.go b/les/vflux/server/balance.go index 727ce09a432fc..b09f7bb5012b0 100644 --- a/les/vflux/server/balance.go +++ b/les/vflux/server/balance.go @@ -356,7 +356,7 @@ func (n *nodeBalance) estimatePriority(capacity uint64, addBalance int64, future b = n.reducedBalance(b, now, future, capacity, avgReqCost) } if bias > 0 { - b = n.reducedBalance(b, now+mclock.AbsTime(future), bias, capacity, 0) + b = n.reducedBalance(b, now.Add(future), bias, capacity, 0) } pri := n.balanceToPriority(now, b, capacity) // Ensure that biased estimates are always lower than actual priorities, even if @@ -512,7 +512,7 @@ func (n *nodeBalance) scheduleCheck(now mclock.AbsTime) { n.updateAfter(0) return } - if n.nextUpdate == 0 || n.nextUpdate > now+mclock.AbsTime(d) { + if n.nextUpdate == 0 || n.nextUpdate > now.Add(d) { if d > time.Second { // Note: if the scheduled update is not in the very near future then we // schedule the update a bit earlier. This way we do need to update a few @@ -520,7 +520,7 @@ func (n *nodeBalance) scheduleCheck(now mclock.AbsTime) { // brings the expected firing time a little bit closer. d = ((d - time.Second) * 7 / 8) + time.Second } - n.nextUpdate = now + mclock.AbsTime(d) + n.nextUpdate = now.Add(d) n.updateAfter(d) } } else { @@ -623,13 +623,13 @@ func (n *nodeBalance) priorityToBalance(priority int64, capacity uint64) (uint64 return 0, uint64(-priority) } -// reducedBalance estimates the reduced balance at a given time in the fututre based +// reducedBalance estimates the reduced balance at a given time in the future based // on the given balance, the time factor and an estimated average request cost per time ratio func (n *nodeBalance) reducedBalance(b balance, start mclock.AbsTime, dt time.Duration, capacity uint64, avgReqCost float64) balance { // since the costs are applied continuously during the dt time period we calculate // the expiration offset at the middle of the period var ( - at = start + mclock.AbsTime(dt/2) + at = start.Add(dt / 2) dtf = float64(dt) ) if !b.pos.IsZero() { diff --git a/les/vflux/server/balance_test.go b/les/vflux/server/balance_test.go index 9f253cabf48db..7c100aab509fa 100644 --- a/les/vflux/server/balance_test.go +++ b/les/vflux/server/balance_test.go @@ -54,7 +54,7 @@ func newBalanceTestSetup(db ethdb.KeyValueStore, posExp, negExp utils.ValueExpir // Initialize and customize the setup for the balance testing clock := &mclock.Simulated{} setup := newServerSetup() - setup.clientField = setup.setup.NewField("balancTestClient", reflect.TypeOf(balanceTestClient{})) + setup.clientField = setup.setup.NewField("balanceTestClient", reflect.TypeOf(balanceTestClient{})) ns := nodestate.NewNodeStateMachine(nil, nil, clock, setup.setup) if posExp == nil { @@ -298,7 +298,7 @@ func TestEstimatedPriority(t *testing.T) { } } -func TestPostiveBalanceCounting(t *testing.T) { +func TestPositiveBalanceCounting(t *testing.T) { b := newBalanceTestSetup(nil, nil, nil) defer b.stop() diff --git a/les/vflux/server/clientpool.go b/les/vflux/server/clientpool.go index e90469bb1c9a4..734d74f453c93 100644 --- a/les/vflux/server/clientpool.go +++ b/les/vflux/server/clientpool.go @@ -61,7 +61,6 @@ type ClientPool struct { setup *serverSetup clock mclock.Clock - closed bool ns *nodestate.NodeStateMachine synced func() bool diff --git a/les/vflux/server/clientpool_test.go b/les/vflux/server/clientpool_test.go index 49e66297a1b1a..790ec51360781 100644 --- a/les/vflux/server/clientpool_test.go +++ b/les/vflux/server/clientpool_test.go @@ -410,7 +410,6 @@ func TestFreeClientKickedOut(t *testing.T) { clock.Run(5 * time.Minute) for i := 0; i < 10; i++ { connect(pool, newPoolTestPeer(i+10, kicked)) - } clock.Run(0) diff --git a/les/vflux/server/status.go b/les/vflux/server/status.go index 469190777b252..2d7e25b684610 100644 --- a/les/vflux/server/status.go +++ b/les/vflux/server/status.go @@ -41,7 +41,7 @@ type serverSetup struct { activeFlag nodestate.Flags // Flag is set if the node is active inactiveFlag nodestate.Flags // Flag is set if the node is inactive capacityField nodestate.Field // Field contains the capacity of the node - queueField nodestate.Field // Field contains the infomration in the priority queue + queueField nodestate.Field // Field contains the information in the priority queue } // newServerSetup initializes the setup for state machine and returns the flags/fields group. diff --git a/light/lightchain.go b/light/lightchain.go index fa0dc71c95996..dca97ce45ce6a 100644 --- a/light/lightchain.go +++ b/light/lightchain.go @@ -355,22 +355,6 @@ func (lc *LightChain) Rollback(chain []common.Hash) { } } -// postChainEvents iterates over the events generated by a chain insertion and -// posts them into the event feed. -func (lc *LightChain) postChainEvents(events []interface{}) { - for _, event := range events { - switch ev := event.(type) { - case core.ChainEvent: - if lc.CurrentHeader().Hash() == ev.Hash { - lc.chainHeadFeed.Send(core.ChainHeadEvent{Block: ev.Block}) - } - lc.chainFeed.Send(ev) - case core.ChainSideEvent: - lc.chainSideFeed.Send(ev) - } - } -} - func (lc *LightChain) InsertHeader(header *types.Header) error { // Verify the header first before obtaining the lock headers := []*types.Header{header} @@ -413,7 +397,7 @@ func (lc *LightChain) SetCanonical(header *types.Header) error { // // The verify parameter can be used to fine tune whether nonce verification // should be done or not. The reason behind the optional check is because some -// of the header retrieval mechanisms already need to verfy nonces, as well as +// of the header retrieval mechanisms already need to verify nonces, as well as // because nonces can be verified sparsely, not needing to check each. // // In the case of a light chain, InsertHeaderChain also creates and posts light diff --git a/light/odr.go b/light/odr.go index 9521dd53e85a9..7cebe010d41fd 100644 --- a/light/odr.go +++ b/light/odr.go @@ -53,9 +53,11 @@ type OdrRequest interface { // TrieID identifies a state or account storage trie type TrieID struct { - BlockHash, Root common.Hash - BlockNumber uint64 - AccKey []byte + BlockHash common.Hash + BlockNumber uint64 + StateRoot common.Hash + Root common.Hash + AccKey []byte } // StateTrieID returns a TrieID for a state trie belonging to a certain block @@ -64,8 +66,9 @@ func StateTrieID(header *types.Header) *TrieID { return &TrieID{ BlockHash: header.Hash(), BlockNumber: header.Number.Uint64(), - AccKey: nil, + StateRoot: header.Root, Root: header.Root, + AccKey: nil, } } @@ -76,6 +79,7 @@ func StorageTrieID(state *TrieID, addrHash, root common.Hash) *TrieID { return &TrieID{ BlockHash: state.BlockHash, BlockNumber: state.BlockNumber, + StateRoot: state.StateRoot, AccKey: addrHash[:], Root: root, } diff --git a/light/odr_test.go b/light/odr_test.go index fdf657a82ec5d..903c7f6f90a6f 100644 --- a/light/odr_test.go +++ b/light/odr_test.go @@ -36,7 +36,6 @@ import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" - "github.com/ethereum/go-ethereum/trie" ) var ( @@ -57,6 +56,7 @@ type testOdr struct { OdrBackend indexerConfig *IndexerConfig sdb, ldb ethdb.Database + serverState state.Database disable bool } @@ -82,7 +82,18 @@ func (odr *testOdr) Retrieve(ctx context.Context, req OdrRequest) error { req.Receipts = rawdb.ReadRawReceipts(odr.sdb, req.Hash, *number) } case *TrieRequest: - t, _ := trie.New(req.Id.Root, trie.NewDatabase(odr.sdb)) + var ( + err error + t state.Trie + ) + if len(req.Id.AccKey) > 0 { + t, err = odr.serverState.OpenStorageTrie(req.Id.StateRoot, common.BytesToHash(req.Id.AccKey), req.Id.Root) + } else { + t, err = odr.serverState.OpenTrie(req.Id.Root) + } + if err != nil { + panic(err) + } nodes := NewNodeSet() t.Prove(req.Key, 0, nodes) req.Proof = nodes @@ -149,7 +160,7 @@ func odrAccounts(ctx context.Context, db ethdb.Database, bc *core.BlockChain, lc st = NewState(ctx, header, lc.Odr()) } else { header := bc.GetHeaderByHash(bhash) - st, _ = state.New(header.Root, state.NewDatabase(db), nil) + st, _ = state.New(header.Root, bc.StateCache(), nil) } var res []byte @@ -189,7 +200,7 @@ func odrContractCall(ctx context.Context, db ethdb.Database, bc *core.BlockChain } else { chain = bc header = bc.GetHeaderByHash(bhash) - st, _ = state.New(header.Root, state.NewDatabase(db), nil) + st, _ = state.New(header.Root, bc.StateCache(), nil) } // Perform read-only call. @@ -253,22 +264,22 @@ func testChainOdr(t *testing.T, protocol int, fn odrTestFn) { var ( sdb = rawdb.NewMemoryDatabase() ldb = rawdb.NewMemoryDatabase() - gspec = core.Genesis{ + gspec = &core.Genesis{ + Config: params.TestChainConfig, Alloc: core.GenesisAlloc{testBankAddress: {Balance: testBankFunds}}, BaseFee: big.NewInt(params.InitialBaseFee), } - genesis = gspec.MustCommit(sdb) ) - gspec.MustCommit(ldb) // Assemble the test environment - blockchain, _ := core.NewBlockChain(sdb, nil, params.TestChainConfig, ethash.NewFullFaker(), vm.Config{}, nil, nil) - gchain, _ := core.GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), sdb, 4, testChainGen) + blockchain, _ := core.NewBlockChain(sdb, nil, gspec, nil, ethash.NewFullFaker(), vm.Config{}, nil, nil) + _, gchain, _ := core.GenerateChainWithGenesis(gspec, ethash.NewFaker(), 4, testChainGen) if _, err := blockchain.InsertChain(gchain); err != nil { t.Fatal(err) } - odr := &testOdr{sdb: sdb, ldb: ldb, indexerConfig: TestClientIndexerConfig} - lightchain, err := NewLightChain(odr, params.TestChainConfig, ethash.NewFullFaker(), nil) + gspec.MustCommit(ldb) + odr := &testOdr{sdb: sdb, ldb: ldb, serverState: blockchain.StateCache(), indexerConfig: TestClientIndexerConfig} + lightchain, err := NewLightChain(odr, gspec.Config, ethash.NewFullFaker(), nil) if err != nil { t.Fatal(err) } diff --git a/light/odr_util.go b/light/odr_util.go index bbbcdbce21359..48631139b488a 100644 --- a/light/odr_util.go +++ b/light/odr_util.go @@ -272,9 +272,9 @@ func GetBloomBits(ctx context.Context, odr OdrBackend, bit uint, sections []uint // GetTransaction retrieves a canonical transaction by hash and also returns // its position in the chain. There is no guarantee in the LES protocol that // the mined transaction will be retrieved back for sure because of different -// reasons(the transaction is unindexed, the malicous server doesn't reply it +// reasons(the transaction is unindexed, the malicious server doesn't reply it // deliberately, etc). Therefore, unretrieved transactions will receive a certain -// number of retrys, thus giving a weak guarantee. +// number of retries, thus giving a weak guarantee. func GetTransaction(ctx context.Context, odr OdrBackend, txHash common.Hash) (*types.Transaction, common.Hash, uint64, uint64, error) { r := &TxStatusRequest{Hashes: []common.Hash{txHash}} if err := odr.RetrieveTxStatus(ctx, r); err != nil || r.Status[0].Status != core.TxStatusIncluded { diff --git a/light/postprocess.go b/light/postprocess.go index ce38d091e891a..bd17eca8a3d21 100644 --- a/light/postprocess.go +++ b/light/postprocess.go @@ -25,7 +25,6 @@ import ( "math/big" "time" - mapset "github.com/deckarep/golang-set" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/bitutil" "github.com/ethereum/go-ethereum/core" @@ -134,7 +133,6 @@ type ChtIndexerBackend struct { diskdb, trieTable ethdb.Database odr OdrBackend triedb *trie.Database - trieset mapset.Set section, sectionSize uint64 lastHash common.Hash trie *trie.Trie @@ -148,7 +146,6 @@ func NewChtIndexer(db ethdb.Database, odr OdrBackend, size, confirms uint64, dis odr: odr, trieTable: trieTable, triedb: trie.NewDatabaseWithConfig(trieTable, &trie.Config{Cache: 1}), // Use a tiny cache only to keep memory down - trieset: mapset.NewSet(), sectionSize: size, disablePruning: disablePruning, } @@ -187,12 +184,12 @@ func (c *ChtIndexerBackend) Reset(ctx context.Context, section uint64, lastSecti root = GetChtRoot(c.diskdb, section-1, lastSectionHead) } var err error - c.trie, err = trie.New(root, c.triedb) + c.trie, err = trie.New(trie.TrieID(root), c.triedb) if err != nil && c.odr != nil { err = c.fetchMissingNodes(ctx, section, root) if err == nil { - c.trie, err = trie.New(root, c.triedb) + c.trie, err = trie.New(trie.TrieID(root), c.triedb) } } c.section = section @@ -217,45 +214,61 @@ func (c *ChtIndexerBackend) Process(ctx context.Context, header *types.Header) e // Commit implements core.ChainIndexerBackend func (c *ChtIndexerBackend) Commit() error { - root, _, err := c.trie.Commit(nil) + root, nodes, err := c.trie.Commit(false) + if err != nil { + return err + } + // Commit trie changes into trie database in case it's not nil. + if nodes != nil { + if err := c.triedb.Update(trie.NewWithNodeSet(nodes)); err != nil { + return err + } + if err := c.triedb.Commit(root, false, nil); err != nil { + return err + } + } + // Re-create trie with newly generated root and updated database. + c.trie, err = trie.New(trie.TrieID(root), c.triedb) if err != nil { return err } // Pruning historical trie nodes if necessary. if !c.disablePruning { - // Flush the triedb and track the latest trie nodes. - c.trieset.Clear() - c.triedb.Commit(root, false, func(hash common.Hash) { c.trieset.Add(hash) }) - it := c.trieTable.NewIterator(nil, nil) defer it.Release() var ( - deleted int - remaining int - t = time.Now() + deleted int + batch = c.trieTable.NewBatch() + t = time.Now() ) + hashes := make(map[common.Hash]struct{}) + if nodes != nil { + for _, hash := range nodes.Hashes() { + hashes[hash] = struct{}{} + } + } for it.Next() { trimmed := bytes.TrimPrefix(it.Key(), []byte(ChtTablePrefix)) - if !c.trieset.Contains(common.BytesToHash(trimmed)) { - c.trieTable.Delete(trimmed) - deleted += 1 - } else { - remaining += 1 + if len(trimmed) == common.HashLength { + if _, ok := hashes[common.BytesToHash(trimmed)]; !ok { + batch.Delete(trimmed) + deleted += 1 + } } } - log.Debug("Prune historical CHT trie nodes", "deleted", deleted, "remaining", remaining, "elapsed", common.PrettyDuration(time.Since(t))) - } else { - c.triedb.Commit(root, false, nil) + if err := batch.Write(); err != nil { + return err + } + log.Debug("Prune historical CHT trie nodes", "deleted", deleted, "remaining", len(hashes), "elapsed", common.PrettyDuration(time.Since(t))) } log.Info("Storing CHT", "section", c.section, "head", fmt.Sprintf("%064x", c.lastHash), "root", fmt.Sprintf("%064x", root)) StoreChtRoot(c.diskdb, c.section, c.lastHash, root) return nil } -// PruneSections implements core.ChainIndexerBackend which deletes all -// chain data(except hash<->number mappings) older than the specified -// threshold. +// Prune implements core.ChainIndexerBackend which deletes all chain data +// (except hash<->number mappings) older than the specified threshold. func (c *ChtIndexerBackend) Prune(threshold uint64) error { // Short circuit if the light pruning is disabled. if c.disablePruning { @@ -303,7 +316,7 @@ var ( BloomTrieTablePrefix = "blt-" ) -// GetBloomTrieRoot reads the BloomTrie root assoctiated to the given section from the database +// GetBloomTrieRoot reads the BloomTrie root associated to the given section from the database func GetBloomTrieRoot(db ethdb.Database, sectionIdx uint64, sectionHead common.Hash) common.Hash { var encNumber [8]byte binary.BigEndian.PutUint64(encNumber[:], sectionIdx) @@ -311,7 +324,7 @@ func GetBloomTrieRoot(db ethdb.Database, sectionIdx uint64, sectionHead common.H return common.BytesToHash(data) } -// StoreBloomTrieRoot writes the BloomTrie root assoctiated to the given section into the database +// StoreBloomTrieRoot writes the BloomTrie root associated to the given section into the database func StoreBloomTrieRoot(db ethdb.Database, sectionIdx uint64, sectionHead, root common.Hash) { var encNumber [8]byte binary.BigEndian.PutUint64(encNumber[:], sectionIdx) @@ -323,7 +336,6 @@ type BloomTrieIndexerBackend struct { disablePruning bool diskdb, trieTable ethdb.Database triedb *trie.Database - trieset mapset.Set odr OdrBackend section uint64 parentSize uint64 @@ -341,7 +353,6 @@ func NewBloomTrieIndexer(db ethdb.Database, odr OdrBackend, parentSize, size uin odr: odr, trieTable: trieTable, triedb: trie.NewDatabaseWithConfig(trieTable, &trie.Config{Cache: 1}), // Use a tiny cache only to keep memory down - trieset: mapset.NewSet(), parentSize: parentSize, size: size, disablePruning: disablePruning, @@ -404,11 +415,11 @@ func (b *BloomTrieIndexerBackend) Reset(ctx context.Context, section uint64, las root = GetBloomTrieRoot(b.diskdb, section-1, lastSectionHead) } var err error - b.trie, err = trie.New(root, b.triedb) + b.trie, err = trie.New(trie.TrieID(root), b.triedb) if err != nil && b.odr != nil { err = b.fetchMissingNodes(ctx, section, root) if err == nil { - b.trie, err = trie.New(root, b.triedb) + b.trie, err = trie.New(trie.TrieID(root), b.triedb) } } b.section = section @@ -454,36 +465,53 @@ func (b *BloomTrieIndexerBackend) Commit() error { b.trie.Delete(encKey[:]) } } - root, _, err := b.trie.Commit(nil) + root, nodes, err := b.trie.Commit(false) + if err != nil { + return err + } + // Commit trie changes into trie database in case it's not nil. + if nodes != nil { + if err := b.triedb.Update(trie.NewWithNodeSet(nodes)); err != nil { + return err + } + if err := b.triedb.Commit(root, false, nil); err != nil { + return err + } + } + // Re-create trie with newly generated root and updated database. + b.trie, err = trie.New(trie.TrieID(root), b.triedb) if err != nil { return err } // Pruning historical trie nodes if necessary. if !b.disablePruning { - // Flush the triedb and track the latest trie nodes. - b.trieset.Clear() - b.triedb.Commit(root, false, func(hash common.Hash) { b.trieset.Add(hash) }) - it := b.trieTable.NewIterator(nil, nil) defer it.Release() var ( - deleted int - remaining int - t = time.Now() + deleted int + batch = b.trieTable.NewBatch() + t = time.Now() ) + hashes := make(map[common.Hash]struct{}) + if nodes != nil { + for _, hash := range nodes.Hashes() { + hashes[hash] = struct{}{} + } + } for it.Next() { trimmed := bytes.TrimPrefix(it.Key(), []byte(BloomTrieTablePrefix)) - if !b.trieset.Contains(common.BytesToHash(trimmed)) { - b.trieTable.Delete(trimmed) - deleted += 1 - } else { - remaining += 1 + if len(trimmed) == common.HashLength { + if _, ok := hashes[common.BytesToHash(trimmed)]; !ok { + batch.Delete(trimmed) + deleted += 1 + } } } - log.Debug("Prune historical bloom trie nodes", "deleted", deleted, "remaining", remaining, "elapsed", common.PrettyDuration(time.Since(t))) - } else { - b.triedb.Commit(root, false, nil) + if err := batch.Write(); err != nil { + return err + } + log.Debug("Prune historical bloom trie nodes", "deleted", deleted, "remaining", len(hashes), "elapsed", common.PrettyDuration(time.Since(t))) } sectionHead := b.sectionHeads[b.bloomTrieRatio-1] StoreBloomTrieRoot(b.diskdb, b.section, sectionHead, root) diff --git a/light/trie.go b/light/trie.go index 4ab6f4ace0756..0092eee136c30 100644 --- a/light/trie.go +++ b/light/trie.go @@ -54,7 +54,7 @@ func (db *odrDatabase) OpenTrie(root common.Hash) (state.Trie, error) { return &odrTrie{db: db, id: db.id}, nil } -func (db *odrDatabase) OpenStorageTrie(addrHash, root common.Hash) (state.Trie, error) { +func (db *odrDatabase) OpenStorageTrie(state, addrHash, root common.Hash) (state.Trie, error) { return &odrTrie{db: db, id: StorageTrieID(db.id, addrHash, root)}, nil } @@ -63,8 +63,7 @@ func (db *odrDatabase) CopyTrie(t state.Trie) state.Trie { case *odrTrie: cpy := &odrTrie{db: t.db, id: t.id} if t.trie != nil { - cpytrie := *t.trie - cpy.trie = &cpytrie + cpy.trie = t.trie.Copy() } return cpy default: @@ -96,6 +95,10 @@ func (db *odrDatabase) TrieDB() *trie.Database { return nil } +func (db *odrDatabase) DiskDB() ethdb.KeyValueStore { + panic("not implemented") +} + type odrTrie struct { db *odrDatabase id *TrieID @@ -112,6 +115,22 @@ func (t *odrTrie) TryGet(key []byte) ([]byte, error) { return res, err } +func (t *odrTrie) TryGetAccount(key []byte) (*types.StateAccount, error) { + key = crypto.Keccak256(key) + var res types.StateAccount + err := t.do(key, func() (err error) { + value, err := t.trie.TryGet(key) + if err != nil { + return err + } + if value == nil { + return nil + } + return rlp.DecodeBytes(value, &res) + }) + return &res, err +} + func (t *odrTrie) TryUpdateAccount(key []byte, acc *types.StateAccount) error { key = crypto.Keccak256(key) value, err := rlp.EncodeToBytes(acc) @@ -137,11 +156,19 @@ func (t *odrTrie) TryDelete(key []byte) error { }) } -func (t *odrTrie) Commit(onleaf trie.LeafCallback) (common.Hash, int, error) { +// TryDeleteAccount abstracts an account deletion from the trie. +func (t *odrTrie) TryDeleteAccount(key []byte) error { + key = crypto.Keccak256(key) + return t.do(key, func() error { + return t.trie.TryDelete(key) + }) +} + +func (t *odrTrie) Commit(collectLeaf bool) (common.Hash, *trie.NodeSet, error) { if t.trie == nil { - return t.id.Root, 0, nil + return t.id.Root, nil, nil } - return t.trie.Commit(onleaf) + return t.trie.Commit(collectLeaf) } func (t *odrTrie) Hash() common.Hash { @@ -169,7 +196,13 @@ func (t *odrTrie) do(key []byte, fn func() error) error { for { var err error if t.trie == nil { - t.trie, err = trie.New(t.id.Root, trie.NewDatabase(t.db.backend.Database())) + var id *trie.ID + if len(t.id.AccKey) > 0 { + id = trie.StorageTrieID(t.id.StateRoot, common.BytesToHash(t.id.AccKey), t.id.Root) + } else { + id = trie.StateTrieID(t.id.StateRoot) + } + t.trie, err = trie.New(id, trie.NewDatabase(t.db.backend.Database())) } if err == nil { err = fn() @@ -195,7 +228,13 @@ func newNodeIterator(t *odrTrie, startkey []byte) trie.NodeIterator { // Open the actual non-ODR trie if that hasn't happened yet. if t.trie == nil { it.do(func() error { - t, err := trie.New(t.id.Root, trie.NewDatabase(t.db.backend.Database())) + var id *trie.ID + if len(t.id.AccKey) > 0 { + id = trie.StorageTrieID(t.id.StateRoot, common.BytesToHash(t.id.AccKey), t.id.Root) + } else { + id = trie.StateTrieID(t.id.StateRoot) + } + t, err := trie.New(id, trie.NewDatabase(t.db.backend.Database())) if err == nil { it.t.trie = t } diff --git a/light/trie_test.go b/light/trie_test.go index e8294cc2a2354..0ab3eb02a0645 100644 --- a/light/trie_test.go +++ b/light/trie_test.go @@ -37,24 +37,24 @@ func TestNodeIterator(t *testing.T) { var ( fulldb = rawdb.NewMemoryDatabase() lightdb = rawdb.NewMemoryDatabase() - gspec = core.Genesis{ + gspec = &core.Genesis{ + Config: params.TestChainConfig, Alloc: core.GenesisAlloc{testBankAddress: {Balance: testBankFunds}}, BaseFee: big.NewInt(params.InitialBaseFee), } - genesis = gspec.MustCommit(fulldb) ) - gspec.MustCommit(lightdb) - blockchain, _ := core.NewBlockChain(fulldb, nil, params.TestChainConfig, ethash.NewFullFaker(), vm.Config{}, nil, nil) - gchain, _ := core.GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), fulldb, 4, testChainGen) + blockchain, _ := core.NewBlockChain(fulldb, nil, gspec, nil, ethash.NewFullFaker(), vm.Config{}, nil, nil) + _, gchain, _ := core.GenerateChainWithGenesis(gspec, ethash.NewFaker(), 4, testChainGen) if _, err := blockchain.InsertChain(gchain); err != nil { panic(err) } + gspec.MustCommit(lightdb) ctx := context.Background() - odr := &testOdr{sdb: fulldb, ldb: lightdb, indexerConfig: TestClientIndexerConfig} + odr := &testOdr{sdb: fulldb, ldb: lightdb, serverState: blockchain.StateCache(), indexerConfig: TestClientIndexerConfig} head := blockchain.CurrentHeader() lightTrie, _ := NewStateDatabase(ctx, head, odr).OpenTrie(head.Root) - fullTrie, _ := state.NewDatabase(fulldb).OpenTrie(head.Root) + fullTrie, _ := blockchain.StateCache().OpenTrie(head.Root) if err := diffTries(fullTrie, lightTrie); err != nil { t.Fatal(err) } @@ -76,7 +76,7 @@ func diffTries(t1, t2 state.Trie) error { case i1.Err != nil: return fmt.Errorf("full trie iterator error: %v", i1.Err) case i2.Err != nil: - return fmt.Errorf("light trie iterator error: %v", i1.Err) + return fmt.Errorf("light trie iterator error: %v", i2.Err) case i1.Next(): return fmt.Errorf("full trie iterator has more k/v pairs") case i2.Next(): diff --git a/light/txpool.go b/light/txpool.go index a7df4aeec3884..0f24fe1bc5150 100644 --- a/light/txpool.go +++ b/light/txpool.go @@ -71,15 +71,16 @@ type TxPool struct { eip2718 bool // Fork indicator whether we are in the eip2718 stage. } -// TxRelayBackend provides an interface to the mechanism that forwards transacions -// to the ETH network. The implementations of the functions should be non-blocking. +// TxRelayBackend provides an interface to the mechanism that forwards transactions to the +// ETH network. The implementations of the functions should be non-blocking. // -// Send instructs backend to forward new transactions -// NewHead notifies backend about a new head after processed by the tx pool, -// including mined and rolled back transactions since the last event -// Discard notifies backend about transactions that should be discarded either -// because they have been replaced by a re-send or because they have been mined -// long ago and no rollback is expected +// Send instructs backend to forward new transactions NewHead notifies backend about a new +// head after processed by the tx pool, including mined and rolled back transactions since +// the last event. +// +// Discard notifies backend about transactions that should be discarded either because +// they have been replaced by a re-send or because they have been mined long ago and no +// rollback is expected. type TxRelayBackend interface { Send(txs types.Transactions) NewHead(head common.Hash, mined []common.Hash, rollback []common.Hash) @@ -398,7 +399,7 @@ func (pool *TxPool) add(ctx context.Context, tx *types.Transaction) error { hash := tx.Hash() if pool.pending[hash] != nil { - return fmt.Errorf("Known transaction (%x)", hash[:4]) + return fmt.Errorf("known transaction (%x)", hash[:4]) } err := pool.validateTx(ctx, tx) if err != nil { diff --git a/light/txpool_test.go b/light/txpool_test.go index cc2651d29ae5d..53732acfa8c81 100644 --- a/light/txpool_test.go +++ b/light/txpool_test.go @@ -83,21 +83,21 @@ func TestTxPool(t *testing.T) { var ( sdb = rawdb.NewMemoryDatabase() ldb = rawdb.NewMemoryDatabase() - gspec = core.Genesis{ + gspec = &core.Genesis{ + Config: params.TestChainConfig, Alloc: core.GenesisAlloc{testBankAddress: {Balance: testBankFunds}}, BaseFee: big.NewInt(params.InitialBaseFee), } - genesis = gspec.MustCommit(sdb) ) - gspec.MustCommit(ldb) // Assemble the test environment - blockchain, _ := core.NewBlockChain(sdb, nil, params.TestChainConfig, ethash.NewFullFaker(), vm.Config{}, nil, nil) - gchain, _ := core.GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), sdb, poolTestBlocks, txPoolTestChainGen) + blockchain, _ := core.NewBlockChain(sdb, nil, gspec, nil, ethash.NewFullFaker(), vm.Config{}, nil, nil) + _, gchain, _ := core.GenerateChainWithGenesis(gspec, ethash.NewFaker(), poolTestBlocks, txPoolTestChainGen) if _, err := blockchain.InsertChain(gchain); err != nil { panic(err) } - odr := &testOdr{sdb: sdb, ldb: ldb, indexerConfig: TestClientIndexerConfig} + gspec.MustCommit(ldb) + odr := &testOdr{sdb: sdb, ldb: ldb, serverState: blockchain.StateCache(), indexerConfig: TestClientIndexerConfig} relay := &testTxRelay{ send: make(chan int, 1), discard: make(chan int, 1), diff --git a/log/doc.go b/log/doc.go index 993743c0fd5c0..d2e15140e4e0e 100644 --- a/log/doc.go +++ b/log/doc.go @@ -7,27 +7,25 @@ This package enforces you to only log key/value pairs. Keys must be strings. Val any type that you like. The default output format is logfmt, but you may also choose to use JSON instead if that suits you. Here's how you log: - log.Info("page accessed", "path", r.URL.Path, "user_id", user.id) + log.Info("page accessed", "path", r.URL.Path, "user_id", user.id) This will output a line that looks like: - lvl=info t=2014-05-02T16:07:23-0700 msg="page accessed" path=/org/71/profile user_id=9 + lvl=info t=2014-05-02T16:07:23-0700 msg="page accessed" path=/org/71/profile user_id=9 -Getting Started +# Getting Started To get started, you'll want to import the library: - import log "github.com/inconshreveable/log15" - + import log "github.com/inconshreveable/log15" Now you're ready to start logging: - func main() { - log.Info("Program starting", "args", os.Args()) - } - + func main() { + log.Info("Program starting", "args", os.Args()) + } -Convention +# Convention Because recording a human-meaningful message is common and good practice, the first argument to every logging method is the value to the *implicit* key 'msg'. @@ -40,38 +38,35 @@ you to favor terseness, ordering, and speed over safety. This is a reasonable tr logging functions. You don't need to explicitly state keys/values, log15 understands that they alternate in the variadic argument list: - log.Warn("size out of bounds", "low", lowBound, "high", highBound, "val", val) + log.Warn("size out of bounds", "low", lowBound, "high", highBound, "val", val) If you really do favor your type-safety, you may choose to pass a log.Ctx instead: - log.Warn("size out of bounds", log.Ctx{"low": lowBound, "high": highBound, "val": val}) - + log.Warn("size out of bounds", log.Ctx{"low": lowBound, "high": highBound, "val": val}) -Context loggers +# Context loggers Frequently, you want to add context to a logger so that you can track actions associated with it. An http request is a good example. You can easily create new loggers that have context that is automatically included with each log line: - requestlogger := log.New("path", r.URL.Path) + requestlogger := log.New("path", r.URL.Path) - // later - requestlogger.Debug("db txn commit", "duration", txnTimer.Finish()) + // later + requestlogger.Debug("db txn commit", "duration", txnTimer.Finish()) This will output a log line that includes the path context that is attached to the logger: - lvl=dbug t=2014-05-02T16:07:23-0700 path=/repo/12/add_hook msg="db txn commit" duration=0.12 + lvl=dbug t=2014-05-02T16:07:23-0700 path=/repo/12/add_hook msg="db txn commit" duration=0.12 - -Handlers +# Handlers The Handler interface defines where log lines are printed to and how they are formatted. Handler is a single interface that is inspired by net/http's handler interface: - type Handler interface { - Log(r *Record) error - } - + type Handler interface { + Log(r *Record) error + } Handlers can filter records, format them, or dispatch to multiple other Handlers. This package implements a number of Handlers for common logging patterns that are @@ -79,49 +74,49 @@ easily composed to create flexible, custom logging structures. Here's an example handler that prints logfmt output to Stdout: - handler := log.StreamHandler(os.Stdout, log.LogfmtFormat()) + handler := log.StreamHandler(os.Stdout, log.LogfmtFormat()) Here's an example handler that defers to two other handlers. One handler only prints records from the rpc package in logfmt to standard out. The other prints records at Error level or above in JSON formatted output to the file /var/log/service.json - handler := log.MultiHandler( - log.LvlFilterHandler(log.LvlError, log.Must.FileHandler("/var/log/service.json", log.JSONFormat())), - log.MatchFilterHandler("pkg", "app/rpc" log.StdoutHandler()) - ) + handler := log.MultiHandler( + log.LvlFilterHandler(log.LvlError, log.Must.FileHandler("/var/log/service.json", log.JSONFormat())), + log.MatchFilterHandler("pkg", "app/rpc" log.StdoutHandler()) + ) -Logging File Names and Line Numbers +# Logging File Names and Line Numbers This package implements three Handlers that add debugging information to the context, CallerFileHandler, CallerFuncHandler and CallerStackHandler. Here's an example that adds the source file and line number of each logging call to the context. - h := log.CallerFileHandler(log.StdoutHandler) - log.Root().SetHandler(h) - ... - log.Error("open file", "err", err) + h := log.CallerFileHandler(log.StdoutHandler) + log.Root().SetHandler(h) + ... + log.Error("open file", "err", err) This will output a line that looks like: - lvl=eror t=2014-05-02T16:07:23-0700 msg="open file" err="file not found" caller=data.go:42 + lvl=eror t=2014-05-02T16:07:23-0700 msg="open file" err="file not found" caller=data.go:42 Here's an example that logs the call stack rather than just the call site. - h := log.CallerStackHandler("%+v", log.StdoutHandler) - log.Root().SetHandler(h) - ... - log.Error("open file", "err", err) + h := log.CallerStackHandler("%+v", log.StdoutHandler) + log.Root().SetHandler(h) + ... + log.Error("open file", "err", err) This will output a line that looks like: - lvl=eror t=2014-05-02T16:07:23-0700 msg="open file" err="file not found" stack="[pkg/data.go:42 pkg/cmd/main.go]" + lvl=eror t=2014-05-02T16:07:23-0700 msg="open file" err="file not found" stack="[pkg/data.go:42 pkg/cmd/main.go]" The "%+v" format instructs the handler to include the path of the source file relative to the compile time GOPATH. The github.com/go-stack/stack package documents the full list of formatting verbs and modifiers available. -Custom Handlers +# Custom Handlers The Handler interface is so simple that it's also trivial to write your own. Let's create an example handler which tries to write to one handler, but if that fails it falls back to @@ -129,24 +124,24 @@ writing to another handler and includes the error that it encountered when tryin to the primary. This might be useful when trying to log over a network socket, but if that fails you want to log those records to a file on disk. - type BackupHandler struct { - Primary Handler - Secondary Handler - } + type BackupHandler struct { + Primary Handler + Secondary Handler + } - func (h *BackupHandler) Log (r *Record) error { - err := h.Primary.Log(r) - if err != nil { - r.Ctx = append(ctx, "primary_err", err) - return h.Secondary.Log(r) - } - return nil - } + func (h *BackupHandler) Log (r *Record) error { + err := h.Primary.Log(r) + if err != nil { + r.Ctx = append(ctx, "primary_err", err) + return h.Secondary.Log(r) + } + return nil + } This pattern is so useful that a generic version that handles an arbitrary number of Handlers is included as part of this library called FailoverHandler. -Logging Expensive Operations +# Logging Expensive Operations Sometimes, you want to log values that are extremely expensive to compute, but you don't want to pay the price of computing them if you haven't turned up your logging level to a high level of detail. @@ -155,50 +150,50 @@ This package provides a simple type to annotate a logging operation that you wan lazily, just when it is about to be logged, so that it would not be evaluated if an upstream Handler filters it out. Just wrap any function which takes no arguments with the log.Lazy type. For example: - func factorRSAKey() (factors []int) { - // return the factors of a very large number - } + func factorRSAKey() (factors []int) { + // return the factors of a very large number + } - log.Debug("factors", log.Lazy{factorRSAKey}) + log.Debug("factors", log.Lazy{factorRSAKey}) If this message is not logged for any reason (like logging at the Error level), then factorRSAKey is never evaluated. -Dynamic context values +# Dynamic context values The same log.Lazy mechanism can be used to attach context to a logger which you want to be evaluated when the message is logged, but not when the logger is created. For example, let's imagine a game where you have Player objects: - type Player struct { - name string - alive bool - log.Logger - } + type Player struct { + name string + alive bool + log.Logger + } You always want to log a player's name and whether they're alive or dead, so when you create the player object, you might do: - p := &Player{name: name, alive: true} - p.Logger = log.New("name", p.name, "alive", p.alive) + p := &Player{name: name, alive: true} + p.Logger = log.New("name", p.name, "alive", p.alive) Only now, even after a player has died, the logger will still report they are alive because the logging context is evaluated when the logger was created. By using the Lazy wrapper, we can defer the evaluation of whether the player is alive or not to each log message, so that the log records will reflect the player's current state no matter when the log message is written: - p := &Player{name: name, alive: true} - isAlive := func() bool { return p.alive } - player.Logger = log.New("name", p.name, "alive", log.Lazy{isAlive}) + p := &Player{name: name, alive: true} + isAlive := func() bool { return p.alive } + player.Logger = log.New("name", p.name, "alive", log.Lazy{isAlive}) -Terminal Format +# Terminal Format If log15 detects that stdout is a terminal, it will configure the default handler for it (which is log.StdoutHandler) to use TerminalFormat. This format logs records nicely for your terminal, including color-coded output based on log level. -Error Handling +# Error Handling Becasuse log15 allows you to step around the type system, there are a few ways you can specify invalid arguments to the logging functions. You could, for example, wrap something that is not @@ -216,61 +211,61 @@ are encouraged to return errors only if they fail to write their log records out syslog daemon is not responding. This allows the construction of useful handlers which cope with those failures like the FailoverHandler. -Library Use +# Library Use log15 is intended to be useful for library authors as a way to provide configurable logging to users of their library. Best practice for use in a library is to always disable all output for your logger by default and to provide a public Logger instance that consumers of your library can configure. Like so: - package yourlib + package yourlib - import "github.com/inconshreveable/log15" + import "github.com/inconshreveable/log15" - var Log = log.New() + var Log = log.New() - func init() { - Log.SetHandler(log.DiscardHandler()) - } + func init() { + Log.SetHandler(log.DiscardHandler()) + } Users of your library may then enable it if they like: - import "github.com/inconshreveable/log15" - import "example.com/yourlib" + import "github.com/inconshreveable/log15" + import "example.com/yourlib" - func main() { - handler := // custom handler setup - yourlib.Log.SetHandler(handler) - } + func main() { + handler := // custom handler setup + yourlib.Log.SetHandler(handler) + } -Best practices attaching logger context +# Best practices attaching logger context The ability to attach context to a logger is a powerful one. Where should you do it and why? I favor embedding a Logger directly into any persistent object in my application and adding unique, tracing context keys to it. For instance, imagine I am writing a web browser: - type Tab struct { - url string - render *RenderingContext - // ... + type Tab struct { + url string + render *RenderingContext + // ... - Logger - } + Logger + } - func NewTab(url string) *Tab { - return &Tab { - // ... - url: url, + func NewTab(url string) *Tab { + return &Tab { + // ... + url: url, - Logger: log.New("url", url), - } - } + Logger: log.New("url", url), + } + } When a new tab is created, I assign a logger to it with the url of the tab as context so it can easily be traced through the logs. Now, whenever we perform any operation with the tab, we'll log with its embedded logger and it will include the tab title automatically: - tab.Debug("moved position", "idx", tab.idx) + tab.Debug("moved position", "idx", tab.idx) There's only one problem. What if the tab url changes? We could use log.Lazy to make sure the current url is always written, but that @@ -285,29 +280,29 @@ function to let you generate what you might call "surrogate keys" They're just random hex identifiers to use for tracing. Back to our Tab example, we would prefer to set up our Logger like so: - import logext "github.com/inconshreveable/log15/ext" + import logext "github.com/inconshreveable/log15/ext" - t := &Tab { - // ... - url: url, - } + t := &Tab { + // ... + url: url, + } - t.Logger = log.New("id", logext.RandId(8), "url", log.Lazy{t.getUrl}) - return t + t.Logger = log.New("id", logext.RandId(8), "url", log.Lazy{t.getUrl}) + return t Now we'll have a unique traceable identifier even across loading new urls, but we'll still be able to see the tab's current url in the log messages. -Must +# Must For all Handler functions which can return an error, there is a version of that function which will return no error but panics on failure. They are all available on the Must object. For example: - log.Must.FileHandler("/path", log.JSONFormat) - log.Must.NetHandler("tcp", ":1234", log.JSONFormat) + log.Must.FileHandler("/path", log.JSONFormat) + log.Must.NetHandler("tcp", ":1234", log.JSONFormat) -Inspiration and Credit +# Inspiration and Credit All of the following excellent projects inspired the design of this library: @@ -325,9 +320,8 @@ github.com/spacemonkeygo/spacelog golang's stdlib, notably io and net/http -The Name +# The Name https://xkcd.com/927/ - */ package log diff --git a/log/format.go b/log/format.go index baf8fddac0f6e..613dc33be769a 100644 --- a/log/format.go +++ b/log/format.go @@ -79,12 +79,11 @@ type TerminalStringer interface { // a terminal with color-coded level output and terser human friendly timestamp. // This format should only be used for interactive programs or while developing. // -// [LEVEL] [TIME] MESSAGE key=value key=value ... +// [LEVEL] [TIME] MESSAGE key=value key=value ... // // Example: // -// [DBUG] [May 16 20:58:45] remove route ns=haproxy addr=127.0.0.1:50002 -// +// [DBUG] [May 16 20:58:45] remove route ns=haproxy addr=127.0.0.1:50002 func TerminalFormat(usecolor bool) Format { return FormatFunc(func(r *Record) []byte { var color = 0 @@ -149,7 +148,6 @@ func TerminalFormat(usecolor bool) Format { // format for key/value pairs. // // For more details see: http://godoc.org/github.com/kr/logfmt -// func LogfmtFormat() Format { return FormatFunc(func(r *Record) []byte { common := []interface{}{r.KeyNames.Time, r.Time, r.KeyNames.Lvl, r.Lvl, r.KeyNames.Msg, r.Msg} diff --git a/log/handler.go b/log/handler.go index 4b9515fa15de0..892cfcc3e1ac5 100644 --- a/log/handler.go +++ b/log/handler.go @@ -136,15 +136,14 @@ func CallerStackHandler(format string, h Handler) Handler { // wrapped Handler if the given function evaluates true. For example, // to only log records where the 'err' key is not nil: // -// logger.SetHandler(FilterHandler(func(r *Record) bool { -// for i := 0; i < len(r.Ctx); i += 2 { -// if r.Ctx[i] == "err" { -// return r.Ctx[i+1] != nil -// } -// } -// return false -// }, h)) -// +// logger.SetHandler(FilterHandler(func(r *Record) bool { +// for i := 0; i < len(r.Ctx); i += 2 { +// if r.Ctx[i] == "err" { +// return r.Ctx[i+1] != nil +// } +// } +// return false +// }, h)) func FilterHandler(fn func(r *Record) bool, h Handler) Handler { return FuncHandler(func(r *Record) error { if fn(r) { @@ -159,8 +158,7 @@ func FilterHandler(fn func(r *Record) bool, h Handler) Handler { // context matches the value. For example, to only log records // from your ui package: // -// log.MatchFilterHandler("pkg", "app/ui", log.StdoutHandler) -// +// log.MatchFilterHandler("pkg", "app/ui", log.StdoutHandler) func MatchFilterHandler(key string, value interface{}, h Handler) Handler { return FilterHandler(func(r *Record) (pass bool) { switch key { @@ -186,8 +184,7 @@ func MatchFilterHandler(key string, value interface{}, h Handler) Handler { // level to the wrapped Handler. For example, to only // log Error/Crit records: // -// log.LvlFilterHandler(log.LvlError, log.StdoutHandler) -// +// log.LvlFilterHandler(log.LvlError, log.StdoutHandler) func LvlFilterHandler(maxLvl Lvl, h Handler) Handler { return FilterHandler(func(r *Record) (pass bool) { return r.Lvl <= maxLvl @@ -199,10 +196,9 @@ func LvlFilterHandler(maxLvl Lvl, h Handler) Handler { // to different locations. For example, to log to a file and // standard error: // -// log.MultiHandler( -// log.Must.FileHandler("/var/log/app.log", log.LogfmtFormat()), -// log.StderrHandler) -// +// log.MultiHandler( +// log.Must.FileHandler("/var/log/app.log", log.LogfmtFormat()), +// log.StderrHandler) func MultiHandler(hs ...Handler) Handler { return FuncHandler(func(r *Record) error { for _, h := range hs { @@ -220,10 +216,10 @@ func MultiHandler(hs ...Handler) Handler { // to writing to a file if the network fails, and then to // standard out if the file write fails: // -// log.FailoverHandler( -// log.Must.NetHandler("tcp", ":9090", log.JSONFormat()), -// log.Must.FileHandler("/var/log/app.log", log.LogfmtFormat()), -// log.StdoutHandler) +// log.FailoverHandler( +// log.Must.NetHandler("tcp", ":9090", log.JSONFormat()), +// log.Must.FileHandler("/var/log/app.log", log.LogfmtFormat()), +// log.StdoutHandler) // // All writes that do not go to the first handler will add context with keys of // the form "failover_err_{idx}" which explain the error encountered while diff --git a/log/handler_glog.go b/log/handler_glog.go index 9b1d4efaf46e8..b5186d4b27ec3 100644 --- a/log/handler_glog.go +++ b/log/handler_glog.go @@ -82,14 +82,14 @@ func (h *GlogHandler) Verbosity(level Lvl) { // // For instance: // -// pattern="gopher.go=3" -// sets the V level to 3 in all Go files named "gopher.go" +// pattern="gopher.go=3" +// sets the V level to 3 in all Go files named "gopher.go" // -// pattern="foo=3" -// sets V to 3 in all files of any packages whose import path ends in "foo" +// pattern="foo=3" +// sets V to 3 in all files of any packages whose import path ends in "foo" // -// pattern="foo/*=3" -// sets V to 3 in all files of any packages whose import path contains "foo" +// pattern="foo/*=3" +// sets V to 3 in all files of any packages whose import path contains "foo" func (h *GlogHandler) Vmodule(ruleset string) error { var filter []pattern for _, rule := range strings.Split(ruleset, ",") { diff --git a/metrics/gauge_float64_test.go b/metrics/gauge_float64_test.go index 02b75580c4e5d..7b854d232ba8a 100644 --- a/metrics/gauge_float64_test.go +++ b/metrics/gauge_float64_test.go @@ -2,7 +2,7 @@ package metrics import "testing" -func BenchmarkGuageFloat64(b *testing.B) { +func BenchmarkGaugeFloat64(b *testing.B) { g := NewGaugeFloat64() b.ResetTimer() for i := 0; i < b.N; i++ { diff --git a/metrics/gauge_test.go b/metrics/gauge_test.go index 3aee143455c37..a98fe985d8c28 100644 --- a/metrics/gauge_test.go +++ b/metrics/gauge_test.go @@ -5,7 +5,7 @@ import ( "testing" ) -func BenchmarkGuage(b *testing.B) { +func BenchmarkGauge(b *testing.B) { g := NewGauge() b.ResetTimer() for i := 0; i < b.N; i++ { diff --git a/metrics/influxdb/influxdb.go b/metrics/influxdb/influxdb.go index dac9e824775a2..e99717aeebf9a 100644 --- a/metrics/influxdb/influxdb.go +++ b/metrics/influxdb/influxdb.go @@ -98,16 +98,16 @@ func (r *reporter) makeClient() (err error) { } func (r *reporter) run() { - intervalTicker := time.Tick(r.interval) - pingTicker := time.Tick(time.Second * 5) + intervalTicker := time.NewTicker(r.interval) + pingTicker := time.NewTicker(time.Second * 5) for { select { - case <-intervalTicker: + case <-intervalTicker.C: if err := r.send(); err != nil { log.Warn("Unable to send to InfluxDB", "err", err) } - case <-pingTicker: + case <-pingTicker.C: _, _, err := r.client.Ping() if err != nil { log.Warn("Got error while sending a ping to InfluxDB, trying to recreate client", "err", err) diff --git a/metrics/influxdb/influxdbv2.go b/metrics/influxdb/influxdbv2.go index 00901f52c9f4f..dc4c04fae16c8 100644 --- a/metrics/influxdb/influxdbv2.go +++ b/metrics/influxdb/influxdbv2.go @@ -1,11 +1,3 @@ -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . package influxdb import ( @@ -67,21 +59,20 @@ func InfluxDBV2WithTags(r metrics.Registry, d time.Duration, endpoint string, to } func (r *v2Reporter) run() { - intervalTicker := time.Tick(r.interval) - pingTicker := time.Tick(time.Second * 5) + intervalTicker := time.NewTicker(r.interval) + pingTicker := time.NewTicker(time.Second * 5) for { select { - case <-intervalTicker: + case <-intervalTicker.C: r.send() - case <-pingTicker: + case <-pingTicker.C: _, err := r.client.Health(context.Background()) if err != nil { log.Warn("Got error from influxdb client health check", "err", err.Error()) } } } - } func (r *v2Reporter) send() { @@ -90,7 +81,6 @@ func (r *v2Reporter) send() { namespace := r.namespace switch metric := i.(type) { - case metrics.Counter: v := metric.Count() l := r.cache[name] diff --git a/metrics/prometheus/prometheus.go b/metrics/prometheus/prometheus.go index 9ad5ec7e9929d..c8408d8cab851 100644 --- a/metrics/prometheus/prometheus.go +++ b/metrics/prometheus/prometheus.go @@ -36,7 +36,7 @@ func Handler(reg metrics.Registry) http.Handler { }) sort.Strings(names) - // Aggregate all the metris into a Prometheus collector + // Aggregate all the metrics into a Prometheus collector c := newCollector() for _, name := range names { diff --git a/metrics/registry_test.go b/metrics/registry_test.go index 6cfedfd88f005..d277ae5c3e477 100644 --- a/metrics/registry_test.go +++ b/metrics/registry_test.go @@ -307,5 +307,4 @@ func TestWalkRegistries(t *testing.T) { if prefix != "prefix.prefix2." { t.Fatal(prefix) } - } diff --git a/miner/miner.go b/miner/miner.go index 16c3bf19d263d..1e9607a76ad9d 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -40,7 +40,6 @@ import ( type Backend interface { BlockChain() *core.BlockChain TxPool() *core.TxPool - StateAtBlock(block *types.Block, reexec uint64, base *state.StateDB, checkLive bool, preferDisk bool) (statedb *state.StateDB, err error) } // Config is the configuration parameters of mining. diff --git a/miner/miner_test.go b/miner/miner_test.go index cf619845dd479..d49c07d964caa 100644 --- a/miner/miner_test.go +++ b/miner/miner_test.go @@ -188,7 +188,6 @@ func TestStartStopMiner(t *testing.T) { waitForMiningState(t, miner, true) miner.Stop() waitForMiningState(t, miner, false) - } func TestCloseMiner(t *testing.T) { @@ -257,7 +256,7 @@ func createMiner(t *testing.T) (*Miner, *event.TypeMux, func(skipMiner bool)) { // Create consensus engine engine := clique.New(chainConfig.Clique, chainDB) // Create Ethereum backend - bc, err := core.NewBlockChain(chainDB, nil, chainConfig, engine, vm.Config{}, nil, nil) + bc, err := core.NewBlockChain(chainDB, nil, genesis, nil, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("can't create new chain %v", err) } diff --git a/miner/stress/beacon/main.go b/miner/stress/beacon/main.go index 3f751049b89e6..88af84c7fcd3b 100644 --- a/miner/stress/beacon/main.go +++ b/miner/stress/beacon/main.go @@ -236,7 +236,7 @@ func newNodeManager(genesis *core.Genesis) *nodeManager { return &nodeManager{ close: make(chan struct{}), genesis: genesis, - genesisBlock: genesis.ToBlock(nil), + genesisBlock: genesis.ToBlock(), } } @@ -316,7 +316,7 @@ func (mgr *nodeManager) run() { nodes := mgr.getNodes(eth2MiningNode) nodes = append(nodes, mgr.getNodes(eth2NormalNode)...) nodes = append(nodes, mgr.getNodes(eth2LightClient)...) - for _, node := range append(nodes) { + for _, node := range nodes { fcState := beacon.ForkchoiceStateV1{ HeadBlockHash: oldest.Hash(), SafeBlockHash: common.Hash{}, diff --git a/miner/unconfirmed_test.go b/miner/unconfirmed_test.go index dc83cb92652d3..60958f658abce 100644 --- a/miner/unconfirmed_test.go +++ b/miner/unconfirmed_test.go @@ -74,7 +74,7 @@ func TestUnconfirmedShifts(t *testing.T) { if n := pool.blocks.Len(); n != int(limit)/2 { t.Errorf("unconfirmed count mismatch: have %d, want %d", n, limit/2) } - // Try to shift all the remaining blocks out and verify emptyness + // Try to shift all the remaining blocks out and verify emptiness pool.Shift(start + 2*uint64(limit)) if n := pool.blocks.Len(); n != 0 { t.Errorf("unconfirmed count mismatch: have %d, want %d", n, 0) diff --git a/miner/worker.go b/miner/worker.go index ae1b61d424111..93fb6288bb45c 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -756,16 +756,6 @@ func (w *worker) makeEnv(parent *types.Block, header *types.Header, coinbase com // Retrieve the parent state to execute on top and start a prefetcher for // the miner to speed block sealing up a bit. state, err := w.chain.StateAt(parent.Root()) - if err != nil { - // Note since the sealing block can be created upon the arbitrary parent - // block, but the state of parent block may already be pruned, so the necessary - // state recovery is needed here in the future. - // - // The maximum acceptable reorg depth can be limited by the finalised block - // somehow. TODO(rjl493456442) fix the hard-coded number here later. - state, err = w.eth.StateAtBlock(parent, 1024, nil, false, false) - log.Warn("Recovered mining state", "root", parent.Root(), "err", err) - } if err != nil { return nil, err } diff --git a/miner/worker_test.go b/miner/worker_test.go index 55361349bcca6..2f1939f75981c 100644 --- a/miner/worker_test.go +++ b/miner/worker_test.go @@ -113,17 +113,15 @@ type testWorkerBackend struct { db ethdb.Database txPool *core.TxPool chain *core.BlockChain - testTxFeed event.Feed genesis *core.Genesis uncleBlock *types.Block } func newTestWorkerBackend(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine, db ethdb.Database, n int) *testWorkerBackend { - var gspec = core.Genesis{ + var gspec = &core.Genesis{ Config: chainConfig, Alloc: core.GenesisAlloc{testBankAddress: {Balance: testBankFunds}}, } - switch e := engine.(type) { case *clique.Clique: gspec.ExtraData = make([]byte, 32+common.AddressLength+crypto.SignatureLength) @@ -135,34 +133,35 @@ func newTestWorkerBackend(t *testing.T, chainConfig *params.ChainConfig, engine default: t.Fatalf("unexpected consensus engine type: %T", engine) } - genesis := gspec.MustCommit(db) - - chain, _ := core.NewBlockChain(db, &core.CacheConfig{TrieDirtyDisabled: true}, gspec.Config, engine, vm.Config{}, nil, nil) + chain, _ := core.NewBlockChain(db, &core.CacheConfig{TrieDirtyDisabled: true}, gspec, nil, engine, vm.Config{}, nil, nil) txpool := core.NewTxPool(testTxPoolConfig, chainConfig, chain) // Generate a small n-block chain and an uncle block for it + var uncle *types.Block if n > 0 { - blocks, _ := core.GenerateChain(chainConfig, genesis, engine, db, n, func(i int, gen *core.BlockGen) { + genDb, blocks, _ := core.GenerateChainWithGenesis(gspec, engine, n, func(i int, gen *core.BlockGen) { gen.SetCoinbase(testBankAddress) }) if _, err := chain.InsertChain(blocks); err != nil { t.Fatalf("failed to insert origin chain: %v", err) } + parent := chain.GetBlockByHash(chain.CurrentBlock().ParentHash()) + blocks, _ = core.GenerateChain(chainConfig, parent, engine, genDb, 1, func(i int, gen *core.BlockGen) { + gen.SetCoinbase(testUserAddress) + }) + uncle = blocks[0] + } else { + _, blocks, _ := core.GenerateChainWithGenesis(gspec, engine, 1, func(i int, gen *core.BlockGen) { + gen.SetCoinbase(testUserAddress) + }) + uncle = blocks[0] } - parent := genesis - if n > 0 { - parent = chain.GetBlockByHash(chain.CurrentBlock().ParentHash()) - } - blocks, _ := core.GenerateChain(chainConfig, parent, engine, db, 1, func(i int, gen *core.BlockGen) { - gen.SetCoinbase(testUserAddress) - }) - return &testWorkerBackend{ db: db, chain: chain, txPool: txpool, - genesis: &gspec, - uncleBlock: blocks[0], + genesis: gspec, + uncleBlock: uncle, } } @@ -218,26 +217,22 @@ func TestGenerateBlockAndImportClique(t *testing.T) { func testGenerateBlockAndImport(t *testing.T, isClique bool) { var ( engine consensus.Engine - chainConfig *params.ChainConfig + chainConfig params.ChainConfig db = rawdb.NewMemoryDatabase() ) if isClique { - chainConfig = params.AllCliqueProtocolChanges + chainConfig = *params.AllCliqueProtocolChanges chainConfig.Clique = ¶ms.CliqueConfig{Period: 1, Epoch: 30000} engine = clique.New(chainConfig.Clique, db) } else { - chainConfig = params.AllEthashProtocolChanges + chainConfig = *params.AllEthashProtocolChanges engine = ethash.NewFaker() } - - chainConfig.LondonBlock = big.NewInt(0) - w, b := newTestWorker(t, chainConfig, engine, db, 0) + w, b := newTestWorker(t, &chainConfig, engine, db, 0) defer w.close() // This test chain imports the mined blocks. - db2 := rawdb.NewMemoryDatabase() - b.genesis.MustCommit(db2) - chain, _ := core.NewBlockChain(db2, nil, b.chain.Config(), engine, vm.Config{}, nil, nil) + chain, _ := core.NewBlockChain(rawdb.NewMemoryDatabase(), nil, b.genesis, nil, engine, vm.Config{}, nil, nil) defer chain.Stop() // Ignore empty commit here for less noise. @@ -495,7 +490,7 @@ func testAdjustInterval(t *testing.T, chainConfig *params.ChainConfig, engine co } w.start() - time.Sleep(time.Second) // Ensure two tasks have been summitted due to start opt + time.Sleep(time.Second) // Ensure two tasks have been submitted due to start opt atomic.StoreUint32(&start, 1) w.setRecommitInterval(3 * time.Second) diff --git a/mobile/accounts.go b/mobile/accounts.go index 4d979bffff5da..d9eab93a741d0 100644 --- a/mobile/accounts.go +++ b/mobile/accounts.go @@ -212,10 +212,10 @@ func (ks *KeyStore) ImportECDSAKey(key []byte, passphrase string) (account *Acco // ImportPreSaleKey decrypts the given Ethereum presale wallet and stores // a key file in the key directory. The key file is encrypted with the same passphrase. -func (ks *KeyStore) ImportPreSaleKey(keyJSON []byte, passphrase string) (ccount *Account, _ error) { - account, err := ks.keystore.ImportPreSaleKey(common.CopyBytes(keyJSON), passphrase) +func (ks *KeyStore) ImportPreSaleKey(keyJSON []byte, passphrase string) (account *Account, _ error) { + acc, err := ks.keystore.ImportPreSaleKey(common.CopyBytes(keyJSON), passphrase) if err != nil { return nil, err } - return &Account{account}, nil + return &Account{acc}, nil } diff --git a/mobile/big.go b/mobile/big.go index c08bcf93f2852..af5f9d89168a1 100644 --- a/mobile/big.go +++ b/mobile/big.go @@ -77,7 +77,6 @@ func (bi *BigInt) SetInt64(x int64) { // -1 if x < 0 // 0 if x == 0 // +1 if x > 0 -// func (bi *BigInt) Sign() int { return bi.bigint.Sign() } diff --git a/mobile/discover.go b/mobile/discover.go index 2c699f08be040..0fbc86de261a0 100644 --- a/mobile/discover.go +++ b/mobile/discover.go @@ -38,8 +38,8 @@ type Enode struct { // // For incomplete nodes, the designator must look like one of these // -// enode:// -// +// enode:// +// // // For complete nodes, the node ID is encoded in the username portion // of the URL, separated from the host by an @ sign. The hostname can @@ -52,7 +52,7 @@ type Enode struct { // a node with IP address 10.3.58.6, TCP listening port 30303 // and UDP discovery port 30301. // -// enode://@10.3.58.6:30303?discport=30301 +// enode://@10.3.58.6:30303?discport=30301 func NewEnode(rawurl string) (*Enode, error) { node, err := enode.Parse(enode.ValidSchemes, rawurl) if err != nil { diff --git a/mobile/doc.go b/mobile/doc.go index 20131afc2ee00..a4d4949ee9235 100644 --- a/mobile/doc.go +++ b/mobile/doc.go @@ -20,7 +20,7 @@ // with pieces plucked from go-ethereum, rather to allow writing native dapps on // mobile platforms. Keep this in mind when using or extending this package! // -// API limitations +// # API limitations // // Since gomobile cannot bridge arbitrary types between Go and Android/iOS, the // exposed APIs need to be manually wrapped into simplified types, with custom diff --git a/mobile/ethclient.go b/mobile/ethclient.go index 662125c4adeb0..00bcb3a2b9bc7 100644 --- a/mobile/ethclient.go +++ b/mobile/ethclient.go @@ -94,7 +94,6 @@ func (ec *EthereumClient) GetTransactionCount(ctx *Context, hash *Hash) (count i func (ec *EthereumClient) GetTransactionInBlock(ctx *Context, hash *Hash, index int) (tx *Transaction, _ error) { rawTx, err := ec.client.TransactionInBlock(ctx.context, hash.hash, uint(index)) return &Transaction{rawTx}, err - } // GetTransactionReceipt returns the receipt of a transaction by transaction hash. diff --git a/mobile/geth.go b/mobile/geth.go index 709b68cbded8b..7dee93b77ca51 100644 --- a/mobile/geth.go +++ b/mobile/geth.go @@ -27,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/ethconfig" + "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/ethstats" "github.com/ethereum/go-ethereum/internal/debug" @@ -35,6 +36,7 @@ import ( "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/nat" "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/rpc" ) // NodeConfig represents the collection of configuration values to fine tune the Geth @@ -156,6 +158,7 @@ func NewNode(datadir string, config *NodeConfig) (stack *Node, _ error) { // Parse the user supplied genesis spec if not mainnet genesis = new(core.Genesis) if err := json.Unmarshal([]byte(config.EthereumGenesis), genesis); err != nil { + rawStack.Close() return nil, fmt.Errorf("invalid genesis spec: %v", err) } // If we have the Ropsten testnet, hard code the chain configs too @@ -196,11 +199,21 @@ func NewNode(datadir string, config *NodeConfig) (stack *Node, _ error) { ethConf.DatabaseCache = config.EthereumDatabaseCache lesBackend, err := les.New(rawStack, ðConf) if err != nil { + rawStack.Close() return nil, fmt.Errorf("ethereum init: %v", err) } + // Register log filter RPC API. + filterSystem := filters.NewFilterSystem(lesBackend.ApiBackend, filters.Config{ + LogCacheSize: ethConf.FilterLogCacheSize, + }) + rawStack.RegisterAPIs([]rpc.API{{ + Namespace: "eth", + Service: filters.NewFilterAPI(filterSystem, true), + }}) // If netstats reporting is requested, do it if config.EthereumNetStats != "" { if err := ethstats.New(rawStack, lesBackend.ApiBackend, lesBackend.Engine(), config.EthereumNetStats); err != nil { + rawStack.Close() return nil, fmt.Errorf("netstats init: %v", err) } } diff --git a/mobile/init.go b/mobile/init.go index 2025d85edc925..94f5baf28be71 100644 --- a/mobile/init.go +++ b/mobile/init.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -// Contains initialization code for the mbile library. +// Contains initialization code for the mobile library. package geth diff --git a/mobile/types.go b/mobile/types.go index a224f12ab23a2..f3f92e4d4ac30 100644 --- a/mobile/types.go +++ b/mobile/types.go @@ -55,7 +55,7 @@ func (n *Nonce) GetBytes() []byte { // GetHex retrieves the hex string representation of the block nonce. func (n *Nonce) GetHex() string { - return fmt.Sprintf("0x%x", n.nonce[:]) + return fmt.Sprintf("%#x", n.nonce[:]) } // String returns a printable representation of the nonce. @@ -75,7 +75,7 @@ func (b *Bloom) GetBytes() []byte { // GetHex retrieves the hex string representation of the bloom filter. func (b *Bloom) GetHex() string { - return fmt.Sprintf("0x%x", b.bloom[:]) + return fmt.Sprintf("%#x", b.bloom[:]) } // String returns a printable representation of the bloom filter. diff --git a/node/api.go b/node/api.go index 1b32399f635c7..67953a812e9dc 100644 --- a/node/api.go +++ b/node/api.go @@ -35,35 +35,26 @@ func (n *Node) apis() []rpc.API { return []rpc.API{ { Namespace: "admin", - Version: "1.0", - Service: &privateAdminAPI{n}, - }, { - Namespace: "admin", - Version: "1.0", - Service: &publicAdminAPI{n}, - Public: true, + Service: &adminAPI{n}, }, { Namespace: "debug", - Version: "1.0", Service: debug.Handler, }, { Namespace: "web3", - Version: "1.0", - Service: &publicWeb3API{n}, - Public: true, + Service: &web3API{n}, }, } } -// privateAdminAPI is the collection of administrative API methods exposed only -// over a secure RPC channel. -type privateAdminAPI struct { +// adminAPI is the collection of administrative API methods exposed over +// both secure and unsecure RPC channels. +type adminAPI struct { node *Node // Node interfaced by this API } // AddPeer requests connecting to a remote node, and also maintaining the new // connection at all times, even reconnecting if it is lost. -func (api *privateAdminAPI) AddPeer(url string) (bool, error) { +func (api *adminAPI) AddPeer(url string) (bool, error) { // Make sure the server is running, fail otherwise server := api.node.Server() if server == nil { @@ -79,7 +70,7 @@ func (api *privateAdminAPI) AddPeer(url string) (bool, error) { } // RemovePeer disconnects from a remote node if the connection exists -func (api *privateAdminAPI) RemovePeer(url string) (bool, error) { +func (api *adminAPI) RemovePeer(url string) (bool, error) { // Make sure the server is running, fail otherwise server := api.node.Server() if server == nil { @@ -95,7 +86,7 @@ func (api *privateAdminAPI) RemovePeer(url string) (bool, error) { } // AddTrustedPeer allows a remote node to always connect, even if slots are full -func (api *privateAdminAPI) AddTrustedPeer(url string) (bool, error) { +func (api *adminAPI) AddTrustedPeer(url string) (bool, error) { // Make sure the server is running, fail otherwise server := api.node.Server() if server == nil { @@ -111,7 +102,7 @@ func (api *privateAdminAPI) AddTrustedPeer(url string) (bool, error) { // RemoveTrustedPeer removes a remote node from the trusted peer set, but it // does not disconnect it automatically. -func (api *privateAdminAPI) RemoveTrustedPeer(url string) (bool, error) { +func (api *adminAPI) RemoveTrustedPeer(url string) (bool, error) { // Make sure the server is running, fail otherwise server := api.node.Server() if server == nil { @@ -127,7 +118,7 @@ func (api *privateAdminAPI) RemoveTrustedPeer(url string) (bool, error) { // PeerEvents creates an RPC subscription which receives peer events from the // node's p2p.Server -func (api *privateAdminAPI) PeerEvents(ctx context.Context) (*rpc.Subscription, error) { +func (api *adminAPI) PeerEvents(ctx context.Context) (*rpc.Subscription, error) { // Make sure the server is running, fail otherwise server := api.node.Server() if server == nil { @@ -164,7 +155,7 @@ func (api *privateAdminAPI) PeerEvents(ctx context.Context) (*rpc.Subscription, } // StartHTTP starts the HTTP RPC API server. -func (api *privateAdminAPI) StartHTTP(host *string, port *int, cors *string, apis *string, vhosts *string) (bool, error) { +func (api *adminAPI) StartHTTP(host *string, port *int, cors *string, apis *string, vhosts *string) (bool, error) { api.node.lock.Lock() defer api.node.lock.Unlock() @@ -219,26 +210,26 @@ func (api *privateAdminAPI) StartHTTP(host *string, port *int, cors *string, api // StartRPC starts the HTTP RPC API server. // Deprecated: use StartHTTP instead. -func (api *privateAdminAPI) StartRPC(host *string, port *int, cors *string, apis *string, vhosts *string) (bool, error) { +func (api *adminAPI) StartRPC(host *string, port *int, cors *string, apis *string, vhosts *string) (bool, error) { log.Warn("Deprecation warning", "method", "admin.StartRPC", "use-instead", "admin.StartHTTP") return api.StartHTTP(host, port, cors, apis, vhosts) } // StopHTTP shuts down the HTTP server. -func (api *privateAdminAPI) StopHTTP() (bool, error) { +func (api *adminAPI) StopHTTP() (bool, error) { api.node.http.stop() return true, nil } // StopRPC shuts down the HTTP server. // Deprecated: use StopHTTP instead. -func (api *privateAdminAPI) StopRPC() (bool, error) { +func (api *adminAPI) StopRPC() (bool, error) { log.Warn("Deprecation warning", "method", "admin.StopRPC", "use-instead", "admin.StopHTTP") return api.StopHTTP() } // StartWS starts the websocket RPC API server. -func (api *privateAdminAPI) StartWS(host *string, port *int, allowedOrigins *string, apis *string) (bool, error) { +func (api *adminAPI) StartWS(host *string, port *int, allowedOrigins *string, apis *string) (bool, error) { api.node.lock.Lock() defer api.node.lock.Unlock() @@ -290,21 +281,15 @@ func (api *privateAdminAPI) StartWS(host *string, port *int, allowedOrigins *str } // StopWS terminates all WebSocket servers. -func (api *privateAdminAPI) StopWS() (bool, error) { +func (api *adminAPI) StopWS() (bool, error) { api.node.http.stopWS() api.node.ws.stop() return true, nil } -// publicAdminAPI is the collection of administrative API methods exposed over -// both secure and unsecure RPC channels. -type publicAdminAPI struct { - node *Node // Node interfaced by this API -} - // Peers retrieves all the information we know about each individual peer at the // protocol granularity. -func (api *publicAdminAPI) Peers() ([]*p2p.PeerInfo, error) { +func (api *adminAPI) Peers() ([]*p2p.PeerInfo, error) { server := api.node.Server() if server == nil { return nil, ErrNodeStopped @@ -314,7 +299,7 @@ func (api *publicAdminAPI) Peers() ([]*p2p.PeerInfo, error) { // NodeInfo retrieves all the information we know about the host node at the // protocol granularity. -func (api *publicAdminAPI) NodeInfo() (*p2p.NodeInfo, error) { +func (api *adminAPI) NodeInfo() (*p2p.NodeInfo, error) { server := api.node.Server() if server == nil { return nil, ErrNodeStopped @@ -323,22 +308,22 @@ func (api *publicAdminAPI) NodeInfo() (*p2p.NodeInfo, error) { } // Datadir retrieves the current data directory the node is using. -func (api *publicAdminAPI) Datadir() string { +func (api *adminAPI) Datadir() string { return api.node.DataDir() } -// publicWeb3API offers helper utils -type publicWeb3API struct { +// web3API offers helper utils +type web3API struct { stack *Node } // ClientVersion returns the node name -func (s *publicWeb3API) ClientVersion() string { +func (s *web3API) ClientVersion() string { return s.stack.Server().Name } // Sha3 applies the ethereum sha3 implementation on the input. // It assumes the input is hex encoded. -func (s *publicWeb3API) Sha3(input hexutil.Bytes) hexutil.Bytes { +func (s *web3API) Sha3(input hexutil.Bytes) hexutil.Bytes { return crypto.Keccak256(input) } diff --git a/node/api_test.go b/node/api_test.go index 9549adf9c254b..d76cb943e4ee9 100644 --- a/node/api_test.go +++ b/node/api_test.go @@ -35,7 +35,7 @@ func TestStartRPC(t *testing.T) { type test struct { name string cfg Config - fn func(*testing.T, *Node, *privateAdminAPI) + fn func(*testing.T, *Node, *adminAPI) // Checks. These run after the node is configured and all API calls have been made. wantReachable bool // whether the HTTP server should be reachable at all @@ -48,7 +48,7 @@ func TestStartRPC(t *testing.T) { { name: "all off", cfg: Config{}, - fn: func(t *testing.T, n *Node, api *privateAdminAPI) { + fn: func(t *testing.T, n *Node, api *adminAPI) { }, wantReachable: false, wantHandlers: false, @@ -58,7 +58,7 @@ func TestStartRPC(t *testing.T) { { name: "rpc enabled through config", cfg: Config{HTTPHost: "127.0.0.1"}, - fn: func(t *testing.T, n *Node, api *privateAdminAPI) { + fn: func(t *testing.T, n *Node, api *adminAPI) { }, wantReachable: true, wantHandlers: true, @@ -68,7 +68,7 @@ func TestStartRPC(t *testing.T) { { name: "rpc enabled through API", cfg: Config{}, - fn: func(t *testing.T, n *Node, api *privateAdminAPI) { + fn: func(t *testing.T, n *Node, api *adminAPI) { _, err := api.StartHTTP(sp("127.0.0.1"), ip(0), nil, nil, nil) assert.NoError(t, err) }, @@ -80,7 +80,7 @@ func TestStartRPC(t *testing.T) { { name: "rpc start again after failure", cfg: Config{}, - fn: func(t *testing.T, n *Node, api *privateAdminAPI) { + fn: func(t *testing.T, n *Node, api *adminAPI) { // Listen on a random port. listener, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { @@ -108,7 +108,7 @@ func TestStartRPC(t *testing.T) { { name: "rpc stopped through API", cfg: Config{HTTPHost: "127.0.0.1"}, - fn: func(t *testing.T, n *Node, api *privateAdminAPI) { + fn: func(t *testing.T, n *Node, api *adminAPI) { _, err := api.StopHTTP() assert.NoError(t, err) }, @@ -120,7 +120,7 @@ func TestStartRPC(t *testing.T) { { name: "rpc stopped twice", cfg: Config{HTTPHost: "127.0.0.1"}, - fn: func(t *testing.T, n *Node, api *privateAdminAPI) { + fn: func(t *testing.T, n *Node, api *adminAPI) { _, err := api.StopHTTP() assert.NoError(t, err) @@ -143,7 +143,7 @@ func TestStartRPC(t *testing.T) { { name: "ws enabled through API", cfg: Config{}, - fn: func(t *testing.T, n *Node, api *privateAdminAPI) { + fn: func(t *testing.T, n *Node, api *adminAPI) { _, err := api.StartWS(sp("127.0.0.1"), ip(0), nil, nil) assert.NoError(t, err) }, @@ -155,7 +155,7 @@ func TestStartRPC(t *testing.T) { { name: "ws stopped through API", cfg: Config{WSHost: "127.0.0.1"}, - fn: func(t *testing.T, n *Node, api *privateAdminAPI) { + fn: func(t *testing.T, n *Node, api *adminAPI) { _, err := api.StopWS() assert.NoError(t, err) }, @@ -167,7 +167,7 @@ func TestStartRPC(t *testing.T) { { name: "ws stopped twice", cfg: Config{WSHost: "127.0.0.1"}, - fn: func(t *testing.T, n *Node, api *privateAdminAPI) { + fn: func(t *testing.T, n *Node, api *adminAPI) { _, err := api.StopWS() assert.NoError(t, err) @@ -182,7 +182,7 @@ func TestStartRPC(t *testing.T) { { name: "ws enabled after RPC", cfg: Config{HTTPHost: "127.0.0.1"}, - fn: func(t *testing.T, n *Node, api *privateAdminAPI) { + fn: func(t *testing.T, n *Node, api *adminAPI) { wsport := n.http.port _, err := api.StartWS(sp("127.0.0.1"), ip(wsport), nil, nil) assert.NoError(t, err) @@ -195,7 +195,7 @@ func TestStartRPC(t *testing.T) { { name: "ws enabled after RPC then stopped", cfg: Config{HTTPHost: "127.0.0.1"}, - fn: func(t *testing.T, n *Node, api *privateAdminAPI) { + fn: func(t *testing.T, n *Node, api *adminAPI) { wsport := n.http.port _, err := api.StartWS(sp("127.0.0.1"), ip(wsport), nil, nil) assert.NoError(t, err) @@ -210,7 +210,7 @@ func TestStartRPC(t *testing.T) { }, { name: "rpc stopped with ws enabled", - fn: func(t *testing.T, n *Node, api *privateAdminAPI) { + fn: func(t *testing.T, n *Node, api *adminAPI) { _, err := api.StartHTTP(sp("127.0.0.1"), ip(0), nil, nil, nil) assert.NoError(t, err) @@ -228,7 +228,7 @@ func TestStartRPC(t *testing.T) { }, { name: "rpc enabled after ws", - fn: func(t *testing.T, n *Node, api *privateAdminAPI) { + fn: func(t *testing.T, n *Node, api *adminAPI) { _, err := api.StartWS(sp("127.0.0.1"), ip(0), nil, nil) assert.NoError(t, err) @@ -271,7 +271,7 @@ func TestStartRPC(t *testing.T) { // Run the API call hook. if test.fn != nil { - test.fn(t, stack, &privateAdminAPI{stack}) + test.fn(t, stack, &adminAPI{stack}) } // Check if the HTTP endpoints are available. diff --git a/node/config.go b/node/config.go index 2047299fb5d74..49959d5ec5de7 100644 --- a/node/config.go +++ b/node/config.go @@ -201,7 +201,7 @@ type Config struct { // AllowUnprotectedTxs allows non EIP-155 protected transactions to be send over RPC. AllowUnprotectedTxs bool `toml:",omitempty"` - // JWTSecret is the hex-encoded jwt secret. + // JWTSecret is the path to the hex-encoded jwt secret. JWTSecret string `toml:",omitempty"` } diff --git a/node/config_test.go b/node/config_test.go index d9f812ec4cb2c..e8af8ddcd87b4 100644 --- a/node/config_test.go +++ b/node/config_test.go @@ -63,12 +63,9 @@ func TestDatadirCreation(t *testing.T) { }() dir = filepath.Join(file.Name(), "invalid/path") - node, err = New(&Config{DataDir: dir}) + _, err = New(&Config{DataDir: dir}) if err == nil { t.Fatalf("protocol stack created with an invalid datadir") - if err := node.Close(); err != nil { - t.Fatalf("failed to close node: %v", err) - } } } @@ -118,7 +115,7 @@ func TestNodeKeyPersistency(t *testing.T) { } config := &Config{Name: "unit-test", DataDir: dir, P2P: p2p.Config{PrivateKey: key}} config.NodeKey() - if _, err := os.Stat(filepath.Join(keyfile)); err == nil { + if _, err := os.Stat(keyfile); err == nil { t.Fatalf("one-shot node key persisted to data directory") } @@ -139,7 +136,7 @@ func TestNodeKeyPersistency(t *testing.T) { // Configure a new node and ensure the previously persisted key is loaded config = &Config{Name: "unit-test", DataDir: dir} config.NodeKey() - blob2, err := os.ReadFile(filepath.Join(keyfile)) + blob2, err := os.ReadFile(keyfile) if err != nil { t.Fatalf("failed to read previously persisted node key: %v", err) } diff --git a/node/doc.go b/node/doc.go index b257f412fed1d..4474e43660d95 100644 --- a/node/doc.go +++ b/node/doc.go @@ -21,25 +21,22 @@ In the model exposed by this package, a node is a collection of services which u resources to provide RPC APIs. Services can also offer devp2p protocols, which are wired up to the devp2p network when the node instance is started. - -Node Lifecycle +# Node Lifecycle The Node object has a lifecycle consisting of three basic states, INITIALIZING, RUNNING and CLOSED. - - ●───────┐ - New() - │ - ▼ - INITIALIZING ────Start()─┐ - │ │ - │ ▼ - Close() RUNNING - │ │ - ▼ │ - CLOSED ◀──────Close()─┘ - + ●───────┐ + New() + │ + ▼ + INITIALIZING ────Start()─┐ + │ │ + │ ▼ + Close() RUNNING + │ │ + ▼ │ + CLOSED ◀──────Close()─┘ Creating a Node allocates basic resources such as the data directory and returns the node in its INITIALIZING state. Lifecycle objects, RPC APIs and peer-to-peer networking @@ -58,8 +55,7 @@ objects and shuts down RPC and peer-to-peer networking. You must always call Close on Node, even if the node was not started. - -Resources Managed By Node +# Resources Managed By Node All file-system resources used by a node instance are located in a directory called the data directory. The location of each resource can be overridden through additional node @@ -83,8 +79,7 @@ without a data directory, databases are opened in memory instead. Node also creates the shared store of encrypted Ethereum account keys. Services can access the account manager through the service context. - -Sharing Data Directory Among Instances +# Sharing Data Directory Among Instances Multiple node instances can share a single data directory if they have distinct instance names (set through the Name config option). Sharing behaviour depends on the type of @@ -102,26 +97,25 @@ create one database for each instance. The account key store is shared among all node instances using the same data directory unless its location is changed through the KeyStoreDir configuration option. - -Data Directory Sharing Example +# Data Directory Sharing Example In this example, two node instances named A and B are started with the same data directory. Node instance A opens the database "db", node instance B opens the databases "db" and "db-2". The following files will be created in the data directory: - data-directory/ - A/ - nodekey -- devp2p node key of instance A - nodes/ -- devp2p discovery knowledge database of instance A - db/ -- LevelDB content for "db" - A.ipc -- JSON-RPC UNIX domain socket endpoint of instance A - B/ - nodekey -- devp2p node key of node B - nodes/ -- devp2p discovery knowledge database of instance B - static-nodes.json -- devp2p static node list of instance B - db/ -- LevelDB content for "db" - db-2/ -- LevelDB content for "db-2" - B.ipc -- JSON-RPC UNIX domain socket endpoint of instance B - keystore/ -- account key store, used by both instances + data-directory/ + A/ + nodekey -- devp2p node key of instance A + nodes/ -- devp2p discovery knowledge database of instance A + db/ -- LevelDB content for "db" + A.ipc -- JSON-RPC UNIX domain socket endpoint of instance A + B/ + nodekey -- devp2p node key of node B + nodes/ -- devp2p discovery knowledge database of instance B + static-nodes.json -- devp2p static node list of instance B + db/ -- LevelDB content for "db" + db-2/ -- LevelDB content for "db-2" + B.ipc -- JSON-RPC UNIX domain socket endpoint of instance B + keystore/ -- account key store, used by both instances */ package node diff --git a/node/endpoints.go b/node/endpoints.go index efc311e7e3173..14c12fd1f175b 100644 --- a/node/endpoints.go +++ b/node/endpoints.go @@ -39,10 +39,11 @@ func StartHTTPEndpoint(endpoint string, timeouts rpc.HTTPTimeouts, handler http. CheckTimeouts(&timeouts) // Bundle and start the HTTP server httpSrv := &http.Server{ - Handler: handler, - ReadTimeout: timeouts.ReadTimeout, - WriteTimeout: timeouts.WriteTimeout, - IdleTimeout: timeouts.IdleTimeout, + Handler: handler, + ReadTimeout: timeouts.ReadTimeout, + ReadHeaderTimeout: timeouts.ReadHeaderTimeout, + WriteTimeout: timeouts.WriteTimeout, + IdleTimeout: timeouts.IdleTimeout, } go httpSrv.Serve(listener) return httpSrv, listener.Addr(), err @@ -75,6 +76,10 @@ func CheckTimeouts(timeouts *rpc.HTTPTimeouts) { log.Warn("Sanitizing invalid HTTP read timeout", "provided", timeouts.ReadTimeout, "updated", rpc.DefaultHTTPTimeouts.ReadTimeout) timeouts.ReadTimeout = rpc.DefaultHTTPTimeouts.ReadTimeout } + if timeouts.ReadHeaderTimeout < time.Second { + log.Warn("Sanitizing invalid HTTP read header timeout", "provided", timeouts.ReadHeaderTimeout, "updated", rpc.DefaultHTTPTimeouts.ReadHeaderTimeout) + timeouts.ReadHeaderTimeout = rpc.DefaultHTTPTimeouts.ReadHeaderTimeout + } if timeouts.WriteTimeout < time.Second { log.Warn("Sanitizing invalid HTTP write timeout", "provided", timeouts.WriteTimeout, "updated", rpc.DefaultHTTPTimeouts.WriteTimeout) timeouts.WriteTimeout = rpc.DefaultHTTPTimeouts.WriteTimeout diff --git a/node/jwt_auth.go b/node/jwt_auth.go new file mode 100644 index 0000000000000..d4f8193ca7f22 --- /dev/null +++ b/node/jwt_auth.go @@ -0,0 +1,45 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package node + +import ( + "fmt" + "net/http" + "time" + + "github.com/ethereum/go-ethereum/rpc" + "github.com/golang-jwt/jwt/v4" +) + +// NewJWTAuth creates an rpc client authentication provider that uses JWT. The +// secret MUST be 32 bytes (256 bits) as defined by the Engine-API authentication spec. +// +// See https://github.com/ethereum/execution-apis/blob/main/src/engine/authentication.md +// for more details about this authentication scheme. +func NewJWTAuth(jwtsecret [32]byte) rpc.HTTPAuth { + return func(h http.Header) error { + token := jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{ + "iat": &jwt.NumericDate{Time: time.Now()}, + }) + s, err := token.SignedString(jwtsecret[:]) + if err != nil { + return fmt.Errorf("failed to create JWT token: %w", err) + } + h.Set("Authorization", "Bearer "+s) + return nil + } +} diff --git a/node/jwt_handler.go b/node/jwt_handler.go index 28d5b87c60bcc..637ae19686e24 100644 --- a/node/jwt_handler.go +++ b/node/jwt_handler.go @@ -24,6 +24,8 @@ import ( "github.com/golang-jwt/jwt/v4" ) +const jwtExpiryTimeout = 60 * time.Second + type jwtHandler struct { keyFunc func(token *jwt.Token) (interface{}, error) next http.Handler @@ -49,7 +51,7 @@ func (handler *jwtHandler) ServeHTTP(out http.ResponseWriter, r *http.Request) { strToken = strings.TrimPrefix(auth, "Bearer ") } if len(strToken) == 0 { - http.Error(out, "missing token", http.StatusForbidden) + http.Error(out, "missing token", http.StatusUnauthorized) return } // We explicitly set only HS256 allowed, and also disables the @@ -61,17 +63,17 @@ func (handler *jwtHandler) ServeHTTP(out http.ResponseWriter, r *http.Request) { switch { case err != nil: - http.Error(out, err.Error(), http.StatusForbidden) + http.Error(out, err.Error(), http.StatusUnauthorized) case !token.Valid: - http.Error(out, "invalid token", http.StatusForbidden) + http.Error(out, "invalid token", http.StatusUnauthorized) case !claims.VerifyExpiresAt(time.Now(), false): // optional - http.Error(out, "token is expired", http.StatusForbidden) + http.Error(out, "token is expired", http.StatusUnauthorized) case claims.IssuedAt == nil: - http.Error(out, "missing issued-at", http.StatusForbidden) - case time.Since(claims.IssuedAt.Time) > 5*time.Second: - http.Error(out, "stale token", http.StatusForbidden) - case time.Until(claims.IssuedAt.Time) > 5*time.Second: - http.Error(out, "future token", http.StatusForbidden) + http.Error(out, "missing issued-at", http.StatusUnauthorized) + case time.Since(claims.IssuedAt.Time) > jwtExpiryTimeout: + http.Error(out, "stale token", http.StatusUnauthorized) + case time.Until(claims.IssuedAt.Time) > jwtExpiryTimeout: + http.Error(out, "future token", http.StatusUnauthorized) default: handler.next.ServeHTTP(out, r) } diff --git a/node/node.go b/node/node.go index 7c540306db2b1..3cbefef022e56 100644 --- a/node/node.go +++ b/node/node.go @@ -20,6 +20,7 @@ import ( crand "crypto/rand" "errors" "fmt" + "hash/crc32" "net/http" "os" "path/filepath" @@ -352,10 +353,10 @@ func (n *Node) obtainJWTSecret(cliParam string) ([]byte, error) { fileName = n.ResolvePath(datadirJWTKey) } // try reading from file - log.Debug("Reading JWT secret", "path", fileName) if data, err := os.ReadFile(fileName); err == nil { jwtSecret := common.FromHex(strings.TrimSpace(string(data))) if len(jwtSecret) == 32 { + log.Info("Loaded JWT secret file", "path", fileName, "crc32", fmt.Sprintf("%#x", crc32.ChecksumIEEE(jwtSecret))) return jwtSecret, nil } log.Error("Invalid JWT secret", "path", fileName, "length", len(jwtSecret)) @@ -667,6 +668,19 @@ func (n *Node) WSEndpoint() string { return "ws://" + n.ws.listenAddr() + n.ws.wsConfig.prefix } +// HTTPAuthEndpoint returns the URL of the authenticated HTTP server. +func (n *Node) HTTPAuthEndpoint() string { + return "http://" + n.httpAuth.listenAddr() +} + +// WSAuthEndpoint returns the current authenticated JSON-RPC over WebSocket endpoint. +func (n *Node) WSAuthEndpoint() string { + if n.httpAuth.wsAllowed() { + return "ws://" + n.httpAuth.listenAddr() + n.httpAuth.wsConfig.prefix + } + return "ws://" + n.wsAuth.listenAddr() + n.wsAuth.wsConfig.prefix +} + // EventMux retrieves the event multiplexer used by all the network services in // the current protocol stack. func (n *Node) EventMux() *event.TypeMux { @@ -702,7 +716,7 @@ func (n *Node) OpenDatabase(name string, cache, handles int, namespace string, r // also attaching a chain freezer to it that moves ancient chain data from the // database to immutable append-only files. If the node is an ephemeral one, a // memory database is returned. -func (n *Node) OpenDatabaseWithFreezer(name string, cache, handles int, freezer, namespace string, readonly bool) (ethdb.Database, error) { +func (n *Node) OpenDatabaseWithFreezer(name string, cache, handles int, ancient string, namespace string, readonly bool) (ethdb.Database, error) { n.lock.Lock() defer n.lock.Unlock() if n.state == closedState { @@ -714,14 +728,7 @@ func (n *Node) OpenDatabaseWithFreezer(name string, cache, handles int, freezer, if n.config.DataDir == "" { db = rawdb.NewMemoryDatabase() } else { - root := n.ResolvePath(name) - switch { - case freezer == "": - freezer = filepath.Join(root, "ancient") - case !filepath.IsAbs(freezer): - freezer = n.ResolvePath(freezer) - } - db, err = rawdb.NewLevelDBDatabaseWithFreezer(root, cache, handles, freezer, namespace, readonly) + db, err = rawdb.NewLevelDBDatabaseWithFreezer(n.ResolvePath(name), cache, handles, n.ResolveAncient(name, ancient), namespace, readonly) } if err == nil { @@ -735,6 +742,17 @@ func (n *Node) ResolvePath(x string) string { return n.config.ResolvePath(x) } +// ResolveAncient returns the absolute path of the root ancient directory. +func (n *Node) ResolveAncient(name string, ancient string) string { + switch { + case ancient == "": + ancient = filepath.Join(n.ResolvePath(name), "ancient") + case !filepath.IsAbs(ancient): + ancient = n.ResolvePath(ancient) + } + return ancient +} + // closeTrackingDB wraps the Close method of a database. When the database is closed by the // service, the wrapper removes it from the node's database map. This ensures that Node // won't auto-close the database if it is closed by the service that opened it. diff --git a/node/node_auth_test.go b/node/node_auth_test.go new file mode 100644 index 0000000000000..597cd8531f790 --- /dev/null +++ b/node/node_auth_test.go @@ -0,0 +1,237 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package node + +import ( + "context" + crand "crypto/rand" + "fmt" + "net/http" + "os" + "path" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/rpc" + "github.com/golang-jwt/jwt/v4" +) + +type helloRPC string + +func (ta helloRPC) HelloWorld() (string, error) { + return string(ta), nil +} + +type authTest struct { + name string + endpoint string + prov rpc.HTTPAuth + expectDialFail bool + expectCall1Fail bool + expectCall2Fail bool +} + +func (at *authTest) Run(t *testing.T) { + ctx := context.Background() + cl, err := rpc.DialOptions(ctx, at.endpoint, rpc.WithHTTPAuth(at.prov)) + if at.expectDialFail { + if err == nil { + t.Fatal("expected initial dial to fail") + } else { + return + } + } + if err != nil { + t.Fatalf("failed to dial rpc endpoint: %v", err) + } + + var x string + err = cl.CallContext(ctx, &x, "engine_helloWorld") + if at.expectCall1Fail { + if err == nil { + t.Fatal("expected call 1 to fail") + } else { + return + } + } + if err != nil { + t.Fatalf("failed to call rpc endpoint: %v", err) + } + if x != "hello engine" { + t.Fatalf("method was silent but did not return expected value: %q", x) + } + + err = cl.CallContext(ctx, &x, "eth_helloWorld") + if at.expectCall2Fail { + if err == nil { + t.Fatal("expected call 2 to fail") + } else { + return + } + } + if err != nil { + t.Fatalf("failed to call rpc endpoint: %v", err) + } + if x != "hello eth" { + t.Fatalf("method was silent but did not return expected value: %q", x) + } +} + +func TestAuthEndpoints(t *testing.T) { + var secret [32]byte + if _, err := crand.Read(secret[:]); err != nil { + t.Fatalf("failed to create jwt secret: %v", err) + } + // Geth must read it from a file, and does not support in-memory JWT secrets, so we create a temporary file. + jwtPath := path.Join(t.TempDir(), "jwt_secret") + if err := os.WriteFile(jwtPath, []byte(hexutil.Encode(secret[:])), 0600); err != nil { + t.Fatalf("failed to prepare jwt secret file: %v", err) + } + // We get ports assigned by the node automatically + conf := &Config{ + HTTPHost: "127.0.0.1", + HTTPPort: 0, + WSHost: "127.0.0.1", + WSPort: 0, + AuthAddr: "127.0.0.1", + AuthPort: 0, + JWTSecret: jwtPath, + + WSModules: []string{"eth", "engine"}, + HTTPModules: []string{"eth", "engine"}, + } + node, err := New(conf) + if err != nil { + t.Fatalf("could not create a new node: %v", err) + } + // register dummy apis so we can test the modules are available and reachable with authentication + node.RegisterAPIs([]rpc.API{ + { + Namespace: "engine", + Version: "1.0", + Service: helloRPC("hello engine"), + Public: true, + Authenticated: true, + }, + { + Namespace: "eth", + Version: "1.0", + Service: helloRPC("hello eth"), + Public: true, + Authenticated: true, + }, + }) + if err := node.Start(); err != nil { + t.Fatalf("failed to start test node: %v", err) + } + defer node.Close() + + // sanity check we are running different endpoints + if a, b := node.WSEndpoint(), node.WSAuthEndpoint(); a == b { + t.Fatalf("expected ws and auth-ws endpoints to be different, got: %q and %q", a, b) + } + if a, b := node.HTTPEndpoint(), node.HTTPAuthEndpoint(); a == b { + t.Fatalf("expected http and auth-http endpoints to be different, got: %q and %q", a, b) + } + + goodAuth := NewJWTAuth(secret) + var otherSecret [32]byte + if _, err := crand.Read(otherSecret[:]); err != nil { + t.Fatalf("failed to create jwt secret: %v", err) + } + badAuth := NewJWTAuth(otherSecret) + + notTooLong := time.Second * 57 + tooLong := time.Second * 60 + requestDelay := time.Second + + testCases := []authTest{ + // Auth works + {name: "ws good", endpoint: node.WSAuthEndpoint(), prov: goodAuth, expectCall1Fail: false}, + {name: "http good", endpoint: node.HTTPAuthEndpoint(), prov: goodAuth, expectCall1Fail: false}, + + // Try a bad auth + {name: "ws bad", endpoint: node.WSAuthEndpoint(), prov: badAuth, expectDialFail: true}, // ws auth is immediate + {name: "http bad", endpoint: node.HTTPAuthEndpoint(), prov: badAuth, expectCall1Fail: true}, // http auth is on first call + + // A common mistake with JWT is to allow the "none" algorithm, which is a valid JWT but not secure. + {name: "ws none", endpoint: node.WSAuthEndpoint(), prov: noneAuth(secret), expectDialFail: true}, + {name: "http none", endpoint: node.HTTPAuthEndpoint(), prov: noneAuth(secret), expectCall1Fail: true}, + + // claims of 5 seconds or more, older or newer, are not allowed + {name: "ws too old", endpoint: node.WSAuthEndpoint(), prov: offsetTimeAuth(secret, -tooLong), expectDialFail: true}, + {name: "http too old", endpoint: node.HTTPAuthEndpoint(), prov: offsetTimeAuth(secret, -tooLong), expectCall1Fail: true}, + // note: for it to be too long we need to add a delay, so that once we receive the request, the difference has not dipped below the "tooLong" + {name: "ws too new", endpoint: node.WSAuthEndpoint(), prov: offsetTimeAuth(secret, tooLong+requestDelay), expectDialFail: true}, + {name: "http too new", endpoint: node.HTTPAuthEndpoint(), prov: offsetTimeAuth(secret, tooLong+requestDelay), expectCall1Fail: true}, + + // Try offset the time, but stay just within bounds + {name: "ws old", endpoint: node.WSAuthEndpoint(), prov: offsetTimeAuth(secret, -notTooLong)}, + {name: "http old", endpoint: node.HTTPAuthEndpoint(), prov: offsetTimeAuth(secret, -notTooLong)}, + {name: "ws new", endpoint: node.WSAuthEndpoint(), prov: offsetTimeAuth(secret, notTooLong)}, + {name: "http new", endpoint: node.HTTPAuthEndpoint(), prov: offsetTimeAuth(secret, notTooLong)}, + + // ws only authenticates on initial dial, then continues communication + {name: "ws single auth", endpoint: node.WSAuthEndpoint(), prov: changingAuth(goodAuth, badAuth)}, + {name: "http call fail auth", endpoint: node.HTTPAuthEndpoint(), prov: changingAuth(goodAuth, badAuth), expectCall2Fail: true}, + {name: "http call fail time", endpoint: node.HTTPAuthEndpoint(), prov: changingAuth(goodAuth, offsetTimeAuth(secret, tooLong+requestDelay)), expectCall2Fail: true}, + } + + for _, testCase := range testCases { + t.Run(testCase.name, testCase.Run) + } +} + +func noneAuth(secret [32]byte) rpc.HTTPAuth { + return func(header http.Header) error { + token := jwt.NewWithClaims(jwt.SigningMethodNone, jwt.MapClaims{ + "iat": &jwt.NumericDate{Time: time.Now()}, + }) + s, err := token.SignedString(secret[:]) + if err != nil { + return fmt.Errorf("failed to create JWT token: %w", err) + } + header.Set("Authorization", "Bearer "+s) + return nil + } +} + +func changingAuth(provs ...rpc.HTTPAuth) rpc.HTTPAuth { + i := 0 + return func(header http.Header) error { + i += 1 + if i > len(provs) { + i = len(provs) + } + return provs[i-1](header) + } +} + +func offsetTimeAuth(secret [32]byte, offset time.Duration) rpc.HTTPAuth { + return func(header http.Header) error { + token := jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{ + "iat": &jwt.NumericDate{Time: time.Now().Add(offset)}, + }) + s, err := token.SignedString(secret[:]) + if err != nil { + return fmt.Errorf("failed to create JWT token: %w", err) + } + header.Set("Authorization", "Bearer "+s) + return nil + } +} diff --git a/node/node_example_test.go b/node/node_example_test.go index d54fe03067df2..e45ee49a25a00 100644 --- a/node/node_example_test.go +++ b/node/node_example_test.go @@ -27,8 +27,8 @@ import ( // life cycle management. // // The following methods are needed to implement a node.Lifecycle: -// - Start() error - method invoked when the node is ready to start the service -// - Stop() error - method invoked when the node terminates the service +// - Start() error - method invoked when the node is ready to start the service +// - Stop() error - method invoked when the node terminates the service type SampleLifecycle struct{} func (s *SampleLifecycle) Start() error { fmt.Println("Service starting..."); return nil } diff --git a/node/node_test.go b/node/node_test.go index 9f9febcacbfee..7c76e21f6baf0 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -581,7 +581,6 @@ func (test rpcPrefixTest) check(t *testing.T, node *Node) { if err == nil { t.Errorf("Error: %s: WebSocket connection succeeded for path in wantNoWS", path) } - } } @@ -614,7 +613,6 @@ func doHTTPRequest(t *testing.T, req *http.Request) *http.Response { resp, err := client.Do(req) if err != nil { t.Fatalf("could not issue a GET request to the given endpoint: %v", err) - } return resp } diff --git a/node/rpcstack.go b/node/rpcstack.go index 0d2be9008a41e..8244c892ff503 100644 --- a/node/rpcstack.go +++ b/node/rpcstack.go @@ -27,6 +27,7 @@ import ( "strings" "sync" "sync/atomic" + "time" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rpc" @@ -81,6 +82,10 @@ type httpServer struct { handlerNames map[string]string } +const ( + shutdownTimeout = 5 * time.Second +) + func newHTTPServer(log log.Logger, timeouts rpc.HTTPTimeouts) *httpServer { h := &httpServer{log: log, timeouts: timeouts, handlerNames: make(map[string]string)} @@ -129,6 +134,7 @@ func (h *httpServer) start() error { if h.timeouts != (rpc.HTTPTimeouts{}) { CheckTimeouts(&h.timeouts) h.server.ReadTimeout = h.timeouts.ReadTimeout + h.server.ReadHeaderTimeout = h.timeouts.ReadHeaderTimeout h.server.WriteTimeout = h.timeouts.WriteTimeout h.server.IdleTimeout = h.timeouts.IdleTimeout } @@ -261,7 +267,15 @@ func (h *httpServer) doStop() { h.wsHandler.Store((*rpcHandler)(nil)) wsHandler.server.Stop() } - h.server.Shutdown(context.Background()) + + ctx, cancel := context.WithTimeout(context.Background(), shutdownTimeout) + defer cancel() + err := h.server.Shutdown(ctx) + if err != nil && err == ctx.Err() { + h.log.Warn("HTTP server graceful shutdown timed out") + h.server.Close() + } + h.listener.Close() h.log.Info("HTTP server stopped", "endpoint", h.listener.Addr()) @@ -281,7 +295,7 @@ func (h *httpServer) enableRPC(apis []rpc.API, config httpConfig) error { // Create RPC server and handler. srv := rpc.NewServer() - if err := RegisterApis(apis, config.Modules, srv, false); err != nil { + if err := RegisterApis(apis, config.Modules, srv); err != nil { return err } h.httpConfig = config @@ -312,7 +326,7 @@ func (h *httpServer) enableWS(apis []rpc.API, config wsConfig) error { } // Create RPC server and handler. srv := rpc.NewServer() - if err := RegisterApis(apis, config.Modules, srv, false); err != nil { + if err := RegisterApis(apis, config.Modules, srv); err != nil { return err } h.wsConfig = config @@ -427,7 +441,6 @@ func (h *virtualHostHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { // It's an IP address, we can serve that h.next.ServeHTTP(w, r) return - } // Not an IP address, but a hostname. Need to validate if _, exist := h.vhosts["*"]; exist { @@ -528,7 +541,7 @@ func (is *ipcServer) stop() error { // RegisterApis checks the given modules' availability, generates an allowlist based on the allowed modules, // and then registers all of the APIs exposed by the services. -func RegisterApis(apis []rpc.API, modules []string, srv *rpc.Server, exposeAll bool) error { +func RegisterApis(apis []rpc.API, modules []string, srv *rpc.Server) error { if bad, available := checkModuleAvailability(modules, apis); len(bad) > 0 { log.Error("Unavailable modules in HTTP API list", "unavailable", bad, "available", available) } @@ -539,7 +552,7 @@ func RegisterApis(apis []rpc.API, modules []string, srv *rpc.Server, exposeAll b } // Register all the APIs exposed by the services for _, api := range apis { - if exposeAll || allowList[api.Namespace] || (len(allowList) == 0 && api.Public) { + if allowList[api.Namespace] || len(allowList) == 0 { if err := srv.RegisterName(api.Namespace, api.Service); err != nil { return err } diff --git a/node/rpcstack_test.go b/node/rpcstack_test.go index 229a5b5e53baa..ebc253800623a 100644 --- a/node/rpcstack_test.go +++ b/node/rpcstack_test.go @@ -100,7 +100,7 @@ func TestWebsocketOrigins(t *testing.T) { expFail: []string{ "test", // no scheme, required by spec "http://test", // wrong scheme - "http://test.foo", "https://a.test.x", // subdomain variatoins + "http://test.foo", "https://a.test.x", // subdomain variations "http://testx:8540", "https://xtest:8540"}, }, // ip tests @@ -319,56 +319,103 @@ func TestJWT(t *testing.T) { wsUrl := fmt.Sprintf("ws://%v", srv.listenAddr()) htUrl := fmt.Sprintf("http://%v", srv.listenAddr()) - expOk := []string{ - fmt.Sprintf("Bearer %v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix()})), - fmt.Sprintf("Bearer %v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix() + 4})), - fmt.Sprintf("Bearer %v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix() - 4})), - fmt.Sprintf("Bearer %v", issueToken(secret, nil, testClaim{ - "iat": time.Now().Unix(), - "exp": time.Now().Unix() + 2, - })), - fmt.Sprintf("Bearer %v", issueToken(secret, nil, testClaim{ - "iat": time.Now().Unix(), - "bar": "baz", - })), + expOk := []func() string{ + func() string { + return fmt.Sprintf("Bearer %v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix()})) + }, + func() string { + return fmt.Sprintf("Bearer %v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix() + 4})) + }, + func() string { + return fmt.Sprintf("Bearer %v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix() - 4})) + }, + func() string { + return fmt.Sprintf("Bearer %v", issueToken(secret, nil, testClaim{ + "iat": time.Now().Unix(), + "exp": time.Now().Unix() + 2, + })) + }, + func() string { + return fmt.Sprintf("Bearer %v", issueToken(secret, nil, testClaim{ + "iat": time.Now().Unix(), + "bar": "baz", + })) + }, } - for i, token := range expOk { + for i, tokenFn := range expOk { + token := tokenFn() if err := wsRequest(t, wsUrl, "Authorization", token); err != nil { t.Errorf("test %d-ws, token '%v': expected ok, got %v", i, token, err) } + token = tokenFn() if resp := rpcRequest(t, htUrl, "Authorization", token); resp.StatusCode != 200 { t.Errorf("test %d-http, token '%v': expected ok, got %v", i, token, resp.StatusCode) } } - expFail := []string{ + + expFail := []func() string{ // future - fmt.Sprintf("Bearer %v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix() + 6})), + func() string { + return fmt.Sprintf("Bearer %v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix() + int64(jwtExpiryTimeout.Seconds()) + 1})) + }, // stale - fmt.Sprintf("Bearer %v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix() - 6})), + func() string { + return fmt.Sprintf("Bearer %v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix() - int64(jwtExpiryTimeout.Seconds()) - 1})) + }, // wrong algo - fmt.Sprintf("Bearer %v", issueToken(secret, jwt.SigningMethodHS512, testClaim{"iat": time.Now().Unix() + 4})), + func() string { + return fmt.Sprintf("Bearer %v", issueToken(secret, jwt.SigningMethodHS512, testClaim{"iat": time.Now().Unix() + 4})) + }, // expired - fmt.Sprintf("Bearer %v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix(), "exp": time.Now().Unix()})), + func() string { + return fmt.Sprintf("Bearer %v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix(), "exp": time.Now().Unix()})) + }, // missing mandatory iat - fmt.Sprintf("Bearer %v", issueToken(secret, nil, testClaim{})), - // wrong secret - fmt.Sprintf("Bearer %v", issueToken([]byte("wrong"), nil, testClaim{"iat": time.Now().Unix()})), - fmt.Sprintf("Bearer %v", issueToken([]byte{}, nil, testClaim{"iat": time.Now().Unix()})), - fmt.Sprintf("Bearer %v", issueToken(nil, nil, testClaim{"iat": time.Now().Unix()})), + func() string { + return fmt.Sprintf("Bearer %v", issueToken(secret, nil, testClaim{})) + }, + // wrong secret + func() string { + return fmt.Sprintf("Bearer %v", issueToken([]byte("wrong"), nil, testClaim{"iat": time.Now().Unix()})) + }, + func() string { + return fmt.Sprintf("Bearer %v", issueToken([]byte{}, nil, testClaim{"iat": time.Now().Unix()})) + }, + func() string { + return fmt.Sprintf("Bearer %v", issueToken(nil, nil, testClaim{"iat": time.Now().Unix()})) + }, // Various malformed syntax - fmt.Sprintf("%v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix()})), - fmt.Sprintf("Bearer %v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix()})), - fmt.Sprintf("bearer %v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix()})), - fmt.Sprintf("Bearer: %v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix()})), - fmt.Sprintf("Bearer:%v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix()})), - fmt.Sprintf("Bearer\t%v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix()})), - fmt.Sprintf("Bearer \t%v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix()})), + func() string { + return fmt.Sprintf("%v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix()})) + }, + func() string { + return fmt.Sprintf("Bearer %v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix()})) + }, + func() string { + return fmt.Sprintf("bearer %v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix()})) + }, + func() string { + return fmt.Sprintf("Bearer: %v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix()})) + }, + func() string { + return fmt.Sprintf("Bearer:%v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix()})) + }, + func() string { + return fmt.Sprintf("Bearer\t%v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix()})) + }, + func() string { + return fmt.Sprintf("Bearer \t%v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix()})) + }, } - for i, token := range expFail { + for i, tokenFn := range expFail { + token := tokenFn() if err := wsRequest(t, wsUrl, "Authorization", token); err == nil { t.Errorf("tc %d-ws, token '%v': expected not to allow, got ok", i, token) } - if resp := rpcRequest(t, htUrl, "Authorization", token); resp.StatusCode != 403 { + + token = tokenFn() + resp := rpcRequest(t, htUrl, "Authorization", token) + if resp.StatusCode != http.StatusUnauthorized { t.Errorf("tc %d-http, token '%v': expected not to allow, got %v", i, token, resp.StatusCode) } } diff --git a/node/utils_test.go b/node/utils_test.go index b7474bb706180..681f3a8b285c4 100644 --- a/node/utils_test.go +++ b/node/utils_test.go @@ -47,8 +47,6 @@ type InstrumentedService struct { startHook func() stopHook func() - - protocols []p2p.Protocol } func (s *InstrumentedService) Start() error { @@ -97,17 +95,12 @@ func (f *FullService) APIs() []rpc.API { return []rpc.API{ { Namespace: "admin", - Version: "1.0", }, { Namespace: "debug", - Version: "1.0", - Public: true, }, { Namespace: "net", - Version: "1.0", - Public: true, }, } } diff --git a/p2p/dial.go b/p2p/dial.go index 0d70e6f4a33b8..02878fae4d311 100644 --- a/p2p/dial.go +++ b/p2p/dial.go @@ -84,13 +84,12 @@ var ( // dialer creates outbound connections and submits them into Server. // Two types of peer connections can be created: // -// - static dials are pre-configured connections. The dialer attempts -// keep these nodes connected at all times. -// -// - dynamic dials are created from node discovery results. The dialer -// continuously reads candidate nodes from its input iterator and attempts -// to create peer connections to nodes arriving through the iterator. +// - static dials are pre-configured connections. The dialer attempts +// keep these nodes connected at all times. // +// - dynamic dials are created from node discovery results. The dialer +// continuously reads candidate nodes from its input iterator and attempts +// to create peer connections to nodes arriving through the iterator. type dialScheduler struct { dialConfig setupFunc dialSetupFunc diff --git a/p2p/discover/lookup.go b/p2p/discover/lookup.go index 9ab4a71ce7b42..b8d97b44e1cc5 100644 --- a/p2p/discover/lookup.go +++ b/p2p/discover/lookup.go @@ -18,6 +18,7 @@ package discover import ( "context" + "errors" "time" "github.com/ethereum/go-ethereum/p2p/enode" @@ -141,7 +142,7 @@ func (it *lookup) slowdown() { func (it *lookup) query(n *node, reply chan<- []*node) { fails := it.tab.db.FindFails(n.ID(), n.IP()) r, err := it.queryfunc(n) - if err == errClosed { + if errors.Is(err, errClosed) { // Avoid recording failures on shutdown. reply <- nil return diff --git a/p2p/discover/ntp.go b/p2p/discover/ntp.go index 1bb52399fbc54..48ceffe95b8db 100644 --- a/p2p/discover/ntp.go +++ b/p2p/discover/ntp.go @@ -108,7 +108,7 @@ func sntpDrift(measurements int) (time.Duration, error) { // Calculate the drift based on an assumed answer time of RRT/2 drifts = append(drifts, sent.Sub(t)+elapsed/2) } - // Calculate average drif (drop two extremities to avoid outliers) + // Calculate average drift (drop two extremities to avoid outliers) sort.Sort(durationSlice(drifts)) drift := time.Duration(0) diff --git a/p2p/discover/table_util_test.go b/p2p/discover/table_util_test.go index 47a2e7ac3caf1..77e03ca9e7e46 100644 --- a/p2p/discover/table_util_test.go +++ b/p2p/discover/table_util_test.go @@ -134,8 +134,8 @@ func newPingRecorder() *pingRecorder { } } -// setRecord updates a node record. Future calls to ping and -// requestENR will return this record. +// updateRecord updates a node record. Future calls to ping and +// RequestENR will return this record. func (t *pingRecorder) updateRecord(n *enode.Node) { t.mu.Lock() defer t.mu.Unlock() @@ -162,7 +162,7 @@ func (t *pingRecorder) ping(n *enode.Node) (seq uint64, err error) { return seq, nil } -// requestENR simulates an ENR request. +// RequestENR simulates an ENR request. func (t *pingRecorder) RequestENR(n *enode.Node) (*enode.Node, error) { t.mu.Lock() defer t.mu.Unlock() diff --git a/p2p/discover/v4_udp.go b/p2p/discover/v4_udp.go index 334716aebed01..67cd2c004cf6e 100644 --- a/p2p/discover/v4_udp.go +++ b/p2p/discover/v4_udp.go @@ -328,13 +328,13 @@ func (t *UDPv4) findnode(toid enode.ID, toaddr *net.UDPAddr, target v4wire.Pubke // enough nodes the reply matcher will time out waiting for the second reply, but // there's no need for an error in that case. err := <-rm.errc - if err == errTimeout && rm.reply != nil { + if errors.Is(err, errTimeout) && rm.reply != nil { err = nil } return nodes, err } -// RequestENR sends enrRequest to the given node and waits for a response. +// RequestENR sends ENRRequest to the given node and waits for a response. func (t *UDPv4) RequestENR(n *enode.Node) (*enode.Node, error) { addr := &net.UDPAddr{IP: n.IP(), Port: n.UDP()} t.ensureBond(n.ID(), addr) @@ -525,8 +525,8 @@ func (t *UDPv4) readLoop(unhandled chan<- ReadPacket) { t.log.Debug("Temporary UDP read error", "err", err) continue } else if err != nil { - // Shut down the loop for permament errors. - if err != io.EOF { + // Shut down the loop for permanent errors. + if !errors.Is(err, io.EOF) { t.log.Debug("UDP read error", "err", err) } return diff --git a/p2p/discover/v4_udp_test.go b/p2p/discover/v4_udp_test.go index e36912f010aeb..f4fd9b246fd39 100644 --- a/p2p/discover/v4_udp_test.go +++ b/p2p/discover/v4_udp_test.go @@ -284,6 +284,7 @@ func TestUDPv4_findnode(t *testing.T) { test.waitPacketOut(func(p *v4wire.Neighbors, to *net.UDPAddr, hash []byte) { if len(p.Nodes) != len(want) { t.Errorf("wrong number of results: got %d, want %d", len(p.Nodes), bucketSize) + return } for i, n := range p.Nodes { if n.ID.ID() != want[i].ID() { @@ -312,7 +313,7 @@ func TestUDPv4_findnodeMultiReply(t *testing.T) { test.table.db.UpdateLastPingReceived(rid, test.remoteaddr.IP, time.Now()) // queue a pending findnode request - resultc, errc := make(chan []*node), make(chan error) + resultc, errc := make(chan []*node, 1), make(chan error, 1) go func() { rid := encodePubkey(&test.remotekey.PublicKey).id() ns, err := test.udp.findnode(rid, test.remoteaddr, testTarget) @@ -489,7 +490,7 @@ func TestUDPv4_EIP868(t *testing.T) { t.Fatalf("invalid record: %v", err) } if !reflect.DeepEqual(n, wantNode) { - t.Fatalf("wrong node in enrResponse: %v", n) + t.Fatalf("wrong node in ENRResponse: %v", n) } }) } diff --git a/p2p/discover/v4wire/v4wire.go b/p2p/discover/v4wire/v4wire.go index d6bf3dc4600af..02ee459d14792 100644 --- a/p2p/discover/v4wire/v4wire.go +++ b/p2p/discover/v4wire/v4wire.go @@ -60,7 +60,7 @@ type ( Pong struct { // This field should mirror the UDP envelope address // of the ping packet, which provides a way to discover the - // the external address (after NAT). + // external address (after NAT). To Endpoint ReplyTok []byte // This contains the hash of the ping packet. Expiration uint64 // Absolute timestamp at which the packet becomes invalid. @@ -86,16 +86,16 @@ type ( Rest []rlp.RawValue `rlp:"tail"` } - // enrRequest queries for the remote node's record. + // ENRRequest queries for the remote node's record. ENRRequest struct { Expiration uint64 // Ignore additional fields (for forward compatibility). Rest []rlp.RawValue `rlp:"tail"` } - // enrResponse is the reply to enrRequest. + // ENRResponse is the reply to ENRRequest. ENRResponse struct { - ReplyTok []byte // Hash of the enrRequest packet. + ReplyTok []byte // Hash of the ENRRequest packet. Record enr.Record // Ignore additional fields (for forward compatibility). Rest []rlp.RawValue `rlp:"tail"` diff --git a/p2p/discover/v5_udp.go b/p2p/discover/v5_udp.go index dc63382fc9011..071ed65adc7f5 100644 --- a/p2p/discover/v5_udp.go +++ b/p2p/discover/v5_udp.go @@ -305,7 +305,7 @@ func (t *UDPv5) lookupWorker(destNode *node, target enode.ID) ([]*node, error) { ) var r []*enode.Node r, err = t.findnode(unwrapNode(destNode), dists) - if err == errClosed { + if errors.Is(err, errClosed) { return nil, err } for _, n := range r { @@ -347,7 +347,7 @@ func (t *UDPv5) ping(n *enode.Node) (uint64, error) { } } -// requestENR requests n's record. +// RequestENR requests n's record. func (t *UDPv5) RequestENR(n *enode.Node) (*enode.Node, error) { nodes, err := t.findnode(n, []uint{0}) if err != nil { @@ -407,6 +407,9 @@ func (t *UDPv5) verifyResponseNode(c *callV5, r *enr.Record, distances []uint, s if err := netutil.CheckRelayIP(c.node.IP(), node.IP()); err != nil { return nil, err } + if t.netrestrict != nil && !t.netrestrict.Contains(node.IP()) { + return nil, errors.New("not contained in netrestrict list") + } if c.node.UDP() <= 1024 { return nil, errLowPort } @@ -622,8 +625,8 @@ func (t *UDPv5) readLoop() { t.log.Debug("Temporary UDP read error", "err", err) continue } else if err != nil { - // Shut down the loop for permament errors. - if err != io.EOF { + // Shut down the loop for permanent errors. + if !errors.Is(err, io.EOF) { t.log.Debug("UDP read error", "err", err) } return diff --git a/p2p/discover/v5wire/encoding.go b/p2p/discover/v5wire/encoding.go index 7d17281ef9691..d605d70803328 100644 --- a/p2p/discover/v5wire/encoding.go +++ b/p2p/discover/v5wire/encoding.go @@ -90,6 +90,10 @@ const ( minVersion = 1 sizeofMaskingIV = 16 + // The minimum size of any Discovery v5 packet is 63 bytes. + // Should reject packets smaller than minPacketSize. + minPacketSize = 63 + minMessageSize = 48 // this refers to data after static headers randomPacketMsgSize = 20 ) @@ -300,7 +304,7 @@ func (c *Codec) encodeWhoareyou(toID enode.ID, packet *Whoareyou) (Header, error return head, nil } -// encodeHandshakeMessage encodes the handshake message packet header. +// encodeHandshakeHeader encodes the handshake message packet header. func (c *Codec) encodeHandshakeHeader(toID enode.ID, addr string, challenge *Whoareyou) (Header, *session, error) { // Ensure calling code sets challenge.node. if challenge.Node == nil { @@ -337,7 +341,7 @@ func (c *Codec) encodeHandshakeHeader(toID enode.ID, addr string, challenge *Who return head, session, err } -// encodeAuthHeader creates the auth header on a request packet following WHOAREYOU. +// makeHandshakeAuth creates the auth header on a request packet following WHOAREYOU. func (c *Codec) makeHandshakeAuth(toID enode.ID, addr string, challenge *Whoareyou) (*handshakeAuthData, *session, error) { auth := new(handshakeAuthData) auth.h.SrcID = c.localnode.ID() @@ -379,7 +383,7 @@ func (c *Codec) makeHandshakeAuth(toID enode.ID, addr string, challenge *Whoarey return auth, sec, err } -// encodeMessage encodes an encrypted message packet. +// encodeMessageHeader encodes an encrypted message packet. func (c *Codec) encodeMessageHeader(toID enode.ID, s *session) (Header, error) { head := c.makeHeader(toID, flagMessage, 0) @@ -415,10 +419,10 @@ func (c *Codec) encryptMessage(s *session, p Packet, head *Header, headerData [] // Decode decodes a discovery packet. func (c *Codec) Decode(input []byte, addr string) (src enode.ID, n *enode.Node, p Packet, err error) { - // Unmask the static header. - if len(input) < sizeofStaticPacketData { + if len(input) < minPacketSize { return enode.ID{}, nil, nil, errTooShort } + // Unmask the static header. var head Header copy(head.IV[:], input[:sizeofMaskingIV]) mask := head.mask(c.localnode.ID()) @@ -596,7 +600,7 @@ func (c *Codec) decodeMessage(fromAddr string, head *Header, headerData, msgData // Try decrypting the message. key := c.sc.readKey(auth.SrcID, fromAddr) msg, err := c.decryptMessage(msgData, head.Nonce[:], headerData, key) - if err == errMessageDecrypt { + if errors.Is(err, errMessageDecrypt) { // It didn't work. Start the handshake since this is an ordinary message packet. return &Unknown{Nonce: head.Nonce}, nil } @@ -632,7 +636,7 @@ func (h *StaticHeader) checkValid(packetLen int) error { return nil } -// headerMask returns a cipher for 'masking' / 'unmasking' packet headers. +// mask returns a cipher for 'masking' / 'unmasking' packet headers. func (h *Header) mask(destID enode.ID) cipher.Stream { block, err := aes.NewCipher(destID[:16]) if err != nil { diff --git a/p2p/discover/v5wire/encoding_test.go b/p2p/discover/v5wire/encoding_test.go index 18aa1db1a41b2..a08cffa2a5766 100644 --- a/p2p/discover/v5wire/encoding_test.go +++ b/p2p/discover/v5wire/encoding_test.go @@ -38,8 +38,7 @@ import ( // To regenerate discv5 test vectors, run // -// go test -run TestVectors -write-test-vectors -// +// go test -run TestVectors -write-test-vectors var writeTestVectorsFlag = flag.Bool("write-test-vectors", false, "Overwrite discv5 test vectors in testdata/") var ( @@ -275,7 +274,15 @@ func TestDecodeErrorsV5(t *testing.T) { net := newHandshakeTest() defer net.close() - net.nodeA.expectDecodeErr(t, errTooShort, []byte{}) + b := make([]byte, 0) + net.nodeA.expectDecodeErr(t, errTooShort, b) + + b = make([]byte, 62) + net.nodeA.expectDecodeErr(t, errTooShort, b) + + b = make([]byte, 63) + net.nodeA.expectDecodeErr(t, errInvalidHeader, b) + // TODO some more tests would be nice :) // - check invalid authdata sizes // - check invalid handshake data sizes diff --git a/p2p/dnsdisc/client.go b/p2p/dnsdisc/client.go index 93868b39a8d4b..3f914d6e94164 100644 --- a/p2p/dnsdisc/client.go +++ b/p2p/dnsdisc/client.go @@ -19,6 +19,7 @@ package dnsdisc import ( "bytes" "context" + "errors" "fmt" "math/rand" "net" @@ -204,7 +205,7 @@ func (c *Client) doResolveEntry(ctx context.Context, domain, hash string) (entry } for _, txt := range txts { e, err := parseEntry(txt, c.cfg.ValidSchemes) - if err == errUnknownEntry { + if errors.Is(err, errUnknownEntry) { continue } if !bytes.HasPrefix(crypto.Keccak256([]byte(txt)), wantHash) { @@ -281,7 +282,7 @@ func (it *randomIterator) nextNode() *enode.Node { } n, err := ct.syncRandom(it.ctx) if err != nil { - if err == it.ctx.Err() { + if errors.Is(err, it.ctx.Err()) { return nil // context canceled. } it.c.cfg.Logger.Debug("Error in DNS random node sync", "tree", ct.loc.domain, "err", err) diff --git a/p2p/dnsdisc/client_test.go b/p2p/dnsdisc/client_test.go index 0a9a96e62167d..93380fdcd3eba 100644 --- a/p2p/dnsdisc/client_test.go +++ b/p2p/dnsdisc/client_test.go @@ -265,7 +265,7 @@ func TestIteratorEmptyTree(t *testing.T) { resolver.add(tree1.ToTXT("n")) // Start the iterator. - node := make(chan *enode.Node) + node := make(chan *enode.Node, 1) it, err := c.NewIterator(url) if err != nil { t.Fatal(err) diff --git a/p2p/dnsdisc/tree.go b/p2p/dnsdisc/tree.go index 7d11e07ef742e..a3f426e428061 100644 --- a/p2p/dnsdisc/tree.go +++ b/p2p/dnsdisc/tree.go @@ -117,32 +117,32 @@ func (t *Tree) Nodes() []*enode.Node { We want to keep the UDP size below 512 bytes. The UDP size is roughly: UDP length = 8 + UDP payload length ( 229 ) UPD Payload length: - - dns.id 2 - - dns.flags 2 - - dns.count.queries 2 - - dns.count.answers 2 - - dns.count.auth_rr 2 - - dns.count.add_rr 2 - - queries (query-size + 6) - - answers : - - dns.resp.name 2 - - dns.resp.type 2 - - dns.resp.class 2 - - dns.resp.ttl 4 - - dns.resp.len 2 - - dns.txt.length 1 - - dns.txt resp_data_size - -So the total size is roughly a fixed overhead of `39`, and the size of the -query (domain name) and response. -The query size is, for example, FVY6INQ6LZ33WLCHO3BPR3FH6Y.snap.mainnet.ethdisco.net (52) + - dns.id 2 + - dns.flags 2 + - dns.count.queries 2 + - dns.count.answers 2 + - dns.count.auth_rr 2 + - dns.count.add_rr 2 + - queries (query-size + 6) + - answers : + - dns.resp.name 2 + - dns.resp.type 2 + - dns.resp.class 2 + - dns.resp.ttl 4 + - dns.resp.len 2 + - dns.txt.length 1 + - dns.txt resp_data_size + +So the total size is roughly a fixed overhead of `39`, and the size of the query (domain +name) and response. The query size is, for example, +FVY6INQ6LZ33WLCHO3BPR3FH6Y.snap.mainnet.ethdisco.net (52) We also have some static data in the response, such as `enrtree-branch:`, and potentially splitting the response up with `" "`, leaving us with a size of roughly `400` that we need to stay below. -The number `370` is used to have some margin for extra overhead (for example, the dns query -may be larger - more subdomains). +The number `370` is used to have some margin for extra overhead (for example, the dns +query may be larger - more subdomains). */ const ( hashAbbrevSize = 1 + 16*13/8 // Size of an encoded hash (plus comma) diff --git a/p2p/enode/urlv4.go b/p2p/enode/urlv4.go index c445049102a70..0272eee98725f 100644 --- a/p2p/enode/urlv4.go +++ b/p2p/enode/urlv4.go @@ -54,8 +54,8 @@ func MustParseV4(rawurl string) *Node { // // For incomplete nodes, the designator must look like one of these // -// enode:// -// +// enode:// +// // // For complete nodes, the node ID is encoded in the username portion // of the URL, separated from the host by an @ sign. The hostname can @@ -68,7 +68,7 @@ func MustParseV4(rawurl string) *Node { // a node with IP address 10.3.58.6, TCP listening port 30303 // and UDP discovery port 30301. // -// enode://@10.3.58.6:30303?discport=30301 +// enode://@10.3.58.6:30303?discport=30301 func ParseV4(rawurl string) (*Node, error) { if m := incompleteNodeURL.FindStringSubmatch(rawurl); m != nil { id, err := parsePubkey(m[1]) diff --git a/p2p/enr/enr.go b/p2p/enr/enr.go index 15891813b41ad..438c7b8a3b36f 100644 --- a/p2p/enr/enr.go +++ b/p2p/enr/enr.go @@ -19,7 +19,7 @@ // stored in key/value pairs. To store and retrieve key/values in a record, use the Entry // interface. // -// Signature Handling +// # Signature Handling // // Records must be signed before transmitting them to another node. // diff --git a/p2p/enr/entries.go b/p2p/enr/entries.go index f2118401afb85..a8b0a3839bda5 100644 --- a/p2p/enr/entries.go +++ b/p2p/enr/entries.go @@ -17,6 +17,7 @@ package enr import ( + "errors" "fmt" "io" "net" @@ -180,9 +181,16 @@ func (err *KeyError) Error() string { return fmt.Sprintf("ENR key %q: %v", err.Key, err.Err) } +func (err *KeyError) Unwrap() error { + return err.Err +} + // IsNotFound reports whether the given error means that a key/value pair is // missing from a record. func IsNotFound(err error) bool { - kerr, ok := err.(*KeyError) - return ok && kerr.Err == errNotFound + var ke *KeyError + if errors.As(err, &ke) { + return ke.Err == errNotFound + } + return false } diff --git a/p2p/message.go b/p2p/message.go index 7cbe0f1dc83e1..24f21456d8e5f 100644 --- a/p2p/message.go +++ b/p2p/message.go @@ -107,12 +107,11 @@ func Send(w MsgWriter, msgcode uint64, data interface{}) error { // SendItems writes an RLP with the given code and data elements. // For a call such as: // -// SendItems(w, code, e1, e2, e3) +// SendItems(w, code, e1, e2, e3) // // the message payload will be an RLP list containing the items: // -// [e1, e2, e3] -// +// [e1, e2, e3] func SendItems(w MsgWriter, msgcode uint64, elems ...interface{}) error { return Send(w, msgcode, elems) } diff --git a/p2p/msgrate/msgrate.go b/p2p/msgrate/msgrate.go index 5bfa27b43378a..adc3758f5b716 100644 --- a/p2p/msgrate/msgrate.go +++ b/p2p/msgrate/msgrate.go @@ -38,14 +38,6 @@ const measurementImpact = 0.1 // to fetch more than some local stable value. const capacityOverestimation = 1.01 -// qosTuningPeers is the number of best peers to tune round trip times based on. -// An Ethereum node doesn't need hundreds of connections to operate correctly, -// so instead of lowering our download speed to the median of potentially many -// bad nodes, we can target a smaller set of vey good nodes. At worse this will -// result in less nodes to sync from, but that's still better than some hogging -// the pipeline. -const qosTuningPeers = 5 - // rttMinEstimate is the minimal round trip time to target requests for. Since // every request entails a 2 way latency + bandwidth + serving database lookups, // it should be generous enough to permit meaningful work to be done on top of @@ -111,7 +103,7 @@ const tuningImpact = 0.25 // local link is saturated. In that case, the live measurements will force us // to reduce request sizes until the throughput gets stable. // -// Lastly, message rate measurements allows us to detect if a peer is unsuaully +// Lastly, message rate measurements allows us to detect if a peer is unusually // slow compared to other peers, in which case we can decide to keep it around // or free up the slot so someone closer. // @@ -127,7 +119,7 @@ type Tracker struct { // in their sizes. // // Callers of course are free to use the item counter as a byte counter if - // or when their protocol of choise if capped by bytes instead of items. + // or when their protocol of choice if capped by bytes instead of items. // (eg. eth.getHeaders vs snap.getAccountRange). capacity map[uint64]float64 @@ -157,7 +149,7 @@ func NewTracker(caps map[uint64]float64, rtt time.Duration) *Tracker { } // Capacity calculates the number of items the peer is estimated to be able to -// retrieve within the alloted time slot. The method will round up any division +// retrieve within the allotted time slot. The method will round up any division // errors and will add an additional overestimation ratio on top. The reason for // overshooting the capacity is because certain message types might not increase // the load proportionally to the requested items, so fetching a bit more might @@ -303,11 +295,15 @@ func (t *Trackers) medianRoundTrip() time.Duration { } sort.Float64s(rtts) - median := rttMaxEstimate - if qosTuningPeers <= len(rtts) { - median = time.Duration(rtts[qosTuningPeers/2]) // Median of our best few peers - } else if len(rtts) > 0 { - median = time.Duration(rtts[len(rtts)/2]) // Median of all out connected peers + var median time.Duration + switch len(rtts) { + case 0: + median = rttMaxEstimate + case 1: + median = time.Duration(rtts[0]) + default: + idx := int(math.Sqrt(float64(len(rtts)))) + median = time.Duration(rtts[idx]) } // Restrict the RTT into some QoS defaults, irrelevant of true RTT if median < rttMinEstimate { diff --git a/p2p/nat/nat.go b/p2p/nat/nat.go index 9d5519b9c4d56..b7c840bc5aee8 100644 --- a/p2p/nat/nat.go +++ b/p2p/nat/nat.go @@ -53,12 +53,12 @@ type Interface interface { // The following formats are currently accepted. // Note that mechanism names are not case-sensitive. // -// "" or "none" return nil -// "extip:77.12.33.4" will assume the local machine is reachable on the given IP -// "any" uses the first auto-detected mechanism -// "upnp" uses the Universal Plug and Play protocol -// "pmp" uses NAT-PMP with an auto-detected gateway address -// "pmp:192.168.0.1" uses NAT-PMP with the given gateway address +// "" or "none" return nil +// "extip:77.12.33.4" will assume the local machine is reachable on the given IP +// "any" uses the first auto-detected mechanism +// "upnp" uses the Universal Plug and Play protocol +// "pmp" uses NAT-PMP with an auto-detected gateway address +// "pmp:192.168.0.1" uses NAT-PMP with the given gateway address func Parse(spec string) (Interface, error) { var ( parts = strings.SplitN(spec, ":", 2) diff --git a/p2p/nat/natupnp.go b/p2p/nat/natupnp.go index 1f5d714664504..a8de00e978b92 100644 --- a/p2p/nat/natupnp.go +++ b/p2p/nat/natupnp.go @@ -79,7 +79,7 @@ func (n *upnp) ExternalIP() (addr net.IP, err error) { func (n *upnp) AddMapping(protocol string, extport, intport int, desc string, lifetime time.Duration) error { ip, err := n.internalAddress() if err != nil { - return nil + return nil // TODO: Shouldn't we return the error? } protocol = strings.ToUpper(protocol) lifetimeS := uint32(lifetime / time.Second) diff --git a/p2p/netutil/error_test.go b/p2p/netutil/error_test.go index 645e48f837412..84d5c2c206211 100644 --- a/p2p/netutil/error_test.go +++ b/p2p/netutil/error_test.go @@ -66,7 +66,6 @@ func TestIsPacketTooBig(t *testing.T) { for i := range buf { if buf[i] != byte(i) { t.Fatalf("error in pattern") - break } } } diff --git a/p2p/peer.go b/p2p/peer.go index 257027a5b74d1..469a1b797416d 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -416,7 +416,7 @@ func (p *Peer) startProtocols(writeStart <-chan struct{}, writeErr chan<- error) if err == nil { p.log.Trace(fmt.Sprintf("Protocol %s/%d returned", proto.Name, proto.Version)) err = errProtocolReturned - } else if err != io.EOF { + } else if !errors.Is(err, io.EOF) { p.log.Trace(fmt.Sprintf("Protocol %s/%d failed", proto.Name, proto.Version), "err", err) } p.protoErr <- err diff --git a/p2p/peer_error.go b/p2p/peer_error.go index 3028685041fe3..ebc59de251a8b 100644 --- a/p2p/peer_error.go +++ b/p2p/peer_error.go @@ -103,7 +103,7 @@ func discReasonForError(err error) DiscReason { if reason, ok := err.(DiscReason); ok { return reason } - if err == errProtocolReturned { + if errors.Is(err, errProtocolReturned) { return DiscQuitting } peerError, ok := err.(*peerError) diff --git a/p2p/server.go b/p2p/server.go index 138975e54bf5f..19f7935ffcaeb 100644 --- a/p2p/server.go +++ b/p2p/server.go @@ -126,7 +126,7 @@ type Config struct { // Protocols should contain the protocols supported // by the server. Matching protocols are launched for // each peer. - Protocols []Protocol `toml:"-"` + Protocols []Protocol `toml:"-" json:"-"` // If ListenAddr is set to a non-nil address, the server // will listen for incoming connections. @@ -136,6 +136,10 @@ type Config struct { // the server is started. ListenAddr string + // If DiscAddr is set to a non-nil value, the server will use ListenAddr + // for TCP and DiscAddr for the UDP discovery protocol. + DiscAddr string + // If set to a non-nil value, the given NAT port mapper // is used to make the listening port available to the // Internet. @@ -549,7 +553,15 @@ func (srv *Server) setupDiscovery() error { return nil } - addr, err := net.ResolveUDPAddr("udp", srv.ListenAddr) + listenAddr := srv.ListenAddr + + // Use an alternate listening address for UDP if + // a custom discovery address is configured. + if srv.DiscAddr != "" { + listenAddr = srv.DiscAddr + } + + addr, err := net.ResolveUDPAddr("udp", listenAddr) if err != nil { return err } diff --git a/p2p/simulations/adapters/exec.go b/p2p/simulations/adapters/exec.go index 35ccdfb068829..7bfa8aab6d103 100644 --- a/p2p/simulations/adapters/exec.go +++ b/p2p/simulations/adapters/exec.go @@ -501,7 +501,6 @@ func startExecNodeStack() (*node.Node, error) { // Add the snapshot API. stack.RegisterAPIs([]rpc.API{{ Namespace: "simulation", - Version: "1.0", Service: SnapshotAPI{services}, }}) diff --git a/p2p/simulations/adapters/types.go b/p2p/simulations/adapters/types.go index aeb8ef77726be..3b4e05a90147f 100644 --- a/p2p/simulations/adapters/types.go +++ b/p2p/simulations/adapters/types.go @@ -39,10 +39,9 @@ import ( // Node represents a node in a simulation network which is created by a // NodeAdapter, for example: // -// * SimNode - An in-memory node -// * ExecNode - A child process node -// * DockerNode - A Docker container node -// +// - SimNode, an in-memory node in the same process +// - ExecNode, a child process node +// - DockerNode, a node running in a Docker container type Node interface { // Addr returns the node's address (e.g. an Enode URL) Addr() []byte diff --git a/p2p/simulations/examples/ping-pong.go b/p2p/simulations/examples/ping-pong.go index 2f4c560548762..d9b51dc09bb06 100644 --- a/p2p/simulations/examples/ping-pong.go +++ b/p2p/simulations/examples/ping-pong.go @@ -139,7 +139,7 @@ const ( func (p *pingPongService) Run(peer *p2p.Peer, rw p2p.MsgReadWriter) error { log := p.log.New("peer.id", peer.ID()) - errC := make(chan error) + errC := make(chan error, 1) go func() { for range time.Tick(10 * time.Second) { log.Info("sending ping") diff --git a/p2p/simulations/http.go b/p2p/simulations/http.go index 341ff8718b7df..b221a0597fc48 100644 --- a/p2p/simulations/http.go +++ b/p2p/simulations/http.go @@ -21,6 +21,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "html" "io" @@ -366,7 +367,6 @@ func (s *Server) StopMocker(w http.ResponseWriter, req *http.Request) { // GetMockerList returns a list of available mockers func (s *Server) GetMockers(w http.ResponseWriter, req *http.Request) { - list := GetMockerList() s.JSON(w, http.StatusOK, list) } @@ -441,6 +441,7 @@ func (s *Server) StreamNetworkEvents(w http.ResponseWriter, req *http.Request) { } } for _, conn := range snap.Conns { + conn := conn event := NewEvent(&conn) if err := writeEvent(event); err != nil { writeErr(err) @@ -559,7 +560,7 @@ func (s *Server) CreateNode(w http.ResponseWriter, req *http.Request) { config := &adapters.NodeConfig{} err := json.NewDecoder(req.Body).Decode(config) - if err != nil && err != io.EOF { + if err != nil && !errors.Is(err, io.EOF) { http.Error(w, err.Error(), http.StatusBadRequest) return } diff --git a/p2p/simulations/http_test.go b/p2p/simulations/http_test.go index f5172f3f23db6..05e43238abb58 100644 --- a/p2p/simulations/http_test.go +++ b/p2p/simulations/http_test.go @@ -489,7 +489,6 @@ func (t *expectEvents) expect(events ...*Event) { } switch expected.Type { - case EventTypeNode: if event.Node == nil { t.Fatal("expected event.Node to be set") @@ -514,7 +513,6 @@ func (t *expectEvents) expect(events ...*Event) { if event.Conn.Up != expected.Conn.Up { t.Fatalf("expected conn event %d to have up=%t, got up=%t", i, expected.Conn.Up, event.Conn.Up) } - } i++ @@ -598,7 +596,7 @@ func TestHTTPSnapshot(t *testing.T) { network, s := testHTTPServer(t) defer s.Close() - var eventsDone = make(chan struct{}) + var eventsDone = make(chan struct{}, 1) count := 1 eventsDoneChan := make(chan *Event) eventSub := network.Events().Subscribe(eventsDoneChan) diff --git a/p2p/simulations/mocker.go b/p2p/simulations/mocker.go index fd25e2c918ddb..47193d83ccb8d 100644 --- a/p2p/simulations/mocker.go +++ b/p2p/simulations/mocker.go @@ -29,20 +29,20 @@ import ( "github.com/ethereum/go-ethereum/p2p/simulations/adapters" ) -//a map of mocker names to its function +// a map of mocker names to its function var mockerList = map[string]func(net *Network, quit chan struct{}, nodeCount int){ "startStop": startStop, "probabilistic": probabilistic, "boot": boot, } -//Lookup a mocker by its name, returns the mockerFn +// Lookup a mocker by its name, returns the mockerFn func LookupMocker(mockerType string) func(net *Network, quit chan struct{}, nodeCount int) { return mockerList[mockerType] } -//Get a list of mockers (keys of the map) -//Useful for frontend to build available mocker selection +// Get a list of mockers (keys of the map) +// Useful for frontend to build available mocker selection func GetMockerList() []string { list := make([]string, 0, len(mockerList)) for k := range mockerList { @@ -51,7 +51,7 @@ func GetMockerList() []string { return list } -//The boot mockerFn only connects the node in a ring and doesn't do anything else +// The boot mockerFn only connects the node in a ring and doesn't do anything else func boot(net *Network, quit chan struct{}, nodeCount int) { _, err := connectNodesInRing(net, nodeCount) if err != nil { @@ -59,7 +59,7 @@ func boot(net *Network, quit chan struct{}, nodeCount int) { } } -//The startStop mockerFn stops and starts nodes in a defined period (ticker) +// The startStop mockerFn stops and starts nodes in a defined period (ticker) func startStop(net *Network, quit chan struct{}, nodeCount int) { nodes, err := connectNodesInRing(net, nodeCount) if err != nil { @@ -96,10 +96,10 @@ func startStop(net *Network, quit chan struct{}, nodeCount int) { } } -//The probabilistic mocker func has a more probabilistic pattern -//(the implementation could probably be improved): -//nodes are connected in a ring, then a varying number of random nodes is selected, -//mocker then stops and starts them in random intervals, and continues the loop +// The probabilistic mocker func has a more probabilistic pattern +// (the implementation could probably be improved): +// nodes are connected in a ring, then a varying number of random nodes is selected, +// mocker then stops and starts them in random intervals, and continues the loop func probabilistic(net *Network, quit chan struct{}, nodeCount int) { nodes, err := connectNodesInRing(net, nodeCount) if err != nil { @@ -157,10 +157,9 @@ func probabilistic(net *Network, quit chan struct{}, nodeCount int) { } wg.Wait() } - } -//connect nodeCount number of nodes in a ring +// connect nodeCount number of nodes in a ring func connectNodesInRing(net *Network, nodeCount int) ([]enode.ID, error) { ids := make([]enode.ID, nodeCount) for i := 0; i < nodeCount; i++ { diff --git a/p2p/simulations/network.go b/p2p/simulations/network.go index 962910dd25bf7..d6c5aca73c5c0 100644 --- a/p2p/simulations/network.go +++ b/p2p/simulations/network.go @@ -235,7 +235,6 @@ func (net *Network) watchPeerEvents(id enode.ID, events chan *p2p.PeerEvent, sub } peer := event.Peer switch event.Type { - case p2p.PeerEventTypeAdd: net.DidConnect(id, peer) @@ -247,7 +246,6 @@ func (net *Network) watchPeerEvents(id enode.ID, events chan *p2p.PeerEvent, sub case p2p.PeerEventTypeMsgRecv: net.DidReceive(peer, id, event.Protocol, *event.MsgCode) - } case err := <-sub.Err(): @@ -927,7 +925,6 @@ func (net *Network) snapshot(addServices []string, removeServices []string) (*Sn if !haveSvc { cleanedServices = append(cleanedServices, svc) } - } snap.Nodes[i].Node.Config.Lifecycles = cleanedServices } @@ -1021,7 +1018,6 @@ func (net *Network) Load(snap *Snapshot) error { // Start connecting. for _, conn := range snap.Conns { - if !net.GetNode(conn.One).Up() || !net.GetNode(conn.Other).Up() { //in this case, at least one of the nodes of a connection is not up, //so it would result in the snapshot `Load` to fail diff --git a/p2p/simulations/network_test.go b/p2p/simulations/network_test.go index fa6936d273c57..ab8cf19462e78 100644 --- a/p2p/simulations/network_test.go +++ b/p2p/simulations/network_test.go @@ -36,7 +36,6 @@ import ( // Tests that a created snapshot with a minimal service only contains the expected connections // and that a network when loaded with this snapshot only contains those same connections func TestSnapshot(t *testing.T) { - // PART I // create snapshot from ring network @@ -204,7 +203,6 @@ OuterTwo: t.Fatal(ctx.Err()) case ev := <-evC: if ev.Type == EventTypeConn && !ev.Control { - // fail on any disconnect if !ev.Conn.Up { t.Fatalf("unexpected disconnect: %v -> %v", ev.Conn.One, ev.Conn.Other) @@ -693,7 +691,6 @@ func BenchmarkMinimalService(b *testing.B) { } func benchmarkMinimalServiceTmp(b *testing.B) { - // stop timer to discard setup time pollution args := strings.Split(b.Name(), "/") nodeCount, err := strconv.ParseInt(args[2], 10, 16) diff --git a/p2p/tracker/tracker.go b/p2p/tracker/tracker.go index 69a49087e2c44..6a733b9ba51e3 100644 --- a/p2p/tracker/tracker.go +++ b/p2p/tracker/tracker.go @@ -121,7 +121,7 @@ func (t *Tracker) Track(peer string, version uint, reqCode uint64, resCode uint6 } // clean is called automatically when a preset time passes without a response -// being dleivered for the first network request. +// being delivered for the first network request. func (t *Tracker) clean() { t.lock.Lock() defer t.lock.Unlock() diff --git a/params/bootnodes.go b/params/bootnodes.go index 2ad230268bc68..b809977745363 100644 --- a/params/bootnodes.go +++ b/params/bootnodes.go @@ -116,6 +116,8 @@ func KnownDNSNetwork(genesis common.Hash, protocol string) string { net = "rinkeby" case GoerliGenesisHash: net = "goerli" + case SepoliaGenesisHash: + net = "sepolia" default: return "" } diff --git a/params/config.go b/params/config.go index 0f8b8a7cb4179..22b36b7d68e3d 100644 --- a/params/config.go +++ b/params/config.go @@ -55,33 +55,38 @@ var CheckpointOracles = map[common.Hash]*CheckpointOracleConfig{ } var ( + MainnetTerminalTotalDifficulty, _ = new(big.Int).SetString("58_750_000_000_000_000_000_000", 0) + // MainnetChainConfig is the chain parameters to run a node on the main network. MainnetChainConfig = &ChainConfig{ - ChainID: big.NewInt(1), - HomesteadBlock: big.NewInt(1_150_000), - DAOForkBlock: big.NewInt(1_920_000), - DAOForkSupport: true, - EIP150Block: big.NewInt(2_463_000), - EIP150Hash: common.HexToHash("0x2086799aeebeae135c246c65021c82b4e15a2c451340993aacfd2751886514f0"), - EIP155Block: big.NewInt(2_675_000), - EIP158Block: big.NewInt(2_675_000), - ByzantiumBlock: big.NewInt(4_370_000), - ConstantinopleBlock: big.NewInt(7_280_000), - PetersburgBlock: big.NewInt(7_280_000), - IstanbulBlock: big.NewInt(9_069_000), - MuirGlacierBlock: big.NewInt(9_200_000), - BerlinBlock: big.NewInt(12_244_000), - LondonBlock: big.NewInt(12_965_000), - ArrowGlacierBlock: big.NewInt(13_773_000), - Ethash: new(EthashConfig), + ChainID: big.NewInt(1), + HomesteadBlock: big.NewInt(1_150_000), + DAOForkBlock: big.NewInt(1_920_000), + DAOForkSupport: true, + EIP150Block: big.NewInt(2_463_000), + EIP150Hash: common.HexToHash("0x2086799aeebeae135c246c65021c82b4e15a2c451340993aacfd2751886514f0"), + EIP155Block: big.NewInt(2_675_000), + EIP158Block: big.NewInt(2_675_000), + ByzantiumBlock: big.NewInt(4_370_000), + ConstantinopleBlock: big.NewInt(7_280_000), + PetersburgBlock: big.NewInt(7_280_000), + IstanbulBlock: big.NewInt(9_069_000), + MuirGlacierBlock: big.NewInt(9_200_000), + BerlinBlock: big.NewInt(12_244_000), + LondonBlock: big.NewInt(12_965_000), + ArrowGlacierBlock: big.NewInt(13_773_000), + GrayGlacierBlock: big.NewInt(15_050_000), + TerminalTotalDifficulty: MainnetTerminalTotalDifficulty, // 58_750_000_000_000_000_000_000 + TerminalTotalDifficultyPassed: true, + Ethash: new(EthashConfig), } // MainnetTrustedCheckpoint contains the light client trusted checkpoint for the main network. MainnetTrustedCheckpoint = &TrustedCheckpoint{ - SectionIndex: 451, - SectionHead: common.HexToHash("0xe47f84b9967eb2ad2afff74d59901b63134660011822fdababaf8fdd18a75aa6"), - CHTRoot: common.HexToHash("0xc31e0462ca3d39a46111bb6b63ac4e1cac84089472b7474a319d582f72b3f0c0"), - BloomRoot: common.HexToHash("0x7c9f25ce3577a3ab330d52a7343f801899cf9d4980c69f81de31ccc1a055c809"), + SectionIndex: 471, + SectionHead: common.HexToHash("0xa03d6354f5ca8d33203bb646ac26a964f240ee54728dcb7483faff0204ec4c9b"), + CHTRoot: common.HexToHash("0x29efeeea3540b7f499b4214d5262bd1fcd87253de10a878f92e6497d848b186f"), + BloomRoot: common.HexToHash("0x2ff6a93ff5e78e823bfc80c6ec856bfe9b20c4ffd0af3cef644a916eabcd3c84"), } // MainnetCheckpointOracle contains a set of configs for the main network oracle. @@ -99,31 +104,32 @@ var ( // RopstenChainConfig contains the chain parameters to run a node on the Ropsten test network. RopstenChainConfig = &ChainConfig{ - ChainID: big.NewInt(3), - HomesteadBlock: big.NewInt(0), - DAOForkBlock: nil, - DAOForkSupport: true, - EIP150Block: big.NewInt(0), - EIP150Hash: common.HexToHash("0x41941023680923e0fe4d74a34bdac8141f2540e3ae90623718e47d66d1ca4a2d"), - EIP155Block: big.NewInt(10), - EIP158Block: big.NewInt(10), - ByzantiumBlock: big.NewInt(1_700_000), - ConstantinopleBlock: big.NewInt(4_230_000), - PetersburgBlock: big.NewInt(4_939_394), - IstanbulBlock: big.NewInt(6_485_846), - MuirGlacierBlock: big.NewInt(7_117_117), - BerlinBlock: big.NewInt(9_812_189), - LondonBlock: big.NewInt(10_499_401), - TerminalTotalDifficulty: new(big.Int).SetBytes([]byte{0x15, 0x2D, 0x02, 0xC7, 0xE1, 0x4A, 0xF6, 0x80, 0x00, 0x00}), // 100_000_000_000_000_000_000_000 - Ethash: new(EthashConfig), + ChainID: big.NewInt(3), + HomesteadBlock: big.NewInt(0), + DAOForkBlock: nil, + DAOForkSupport: true, + EIP150Block: big.NewInt(0), + EIP150Hash: common.HexToHash("0x41941023680923e0fe4d74a34bdac8141f2540e3ae90623718e47d66d1ca4a2d"), + EIP155Block: big.NewInt(10), + EIP158Block: big.NewInt(10), + ByzantiumBlock: big.NewInt(1_700_000), + ConstantinopleBlock: big.NewInt(4_230_000), + PetersburgBlock: big.NewInt(4_939_394), + IstanbulBlock: big.NewInt(6_485_846), + MuirGlacierBlock: big.NewInt(7_117_117), + BerlinBlock: big.NewInt(9_812_189), + LondonBlock: big.NewInt(10_499_401), + TerminalTotalDifficulty: new(big.Int).SetUint64(50_000_000_000_000_000), + TerminalTotalDifficultyPassed: true, + Ethash: new(EthashConfig), } // RopstenTrustedCheckpoint contains the light client trusted checkpoint for the Ropsten test network. RopstenTrustedCheckpoint = &TrustedCheckpoint{ - SectionIndex: 346, - SectionHead: common.HexToHash("0xafa0384ebd13a751fb7475aaa7fc08ac308925c8b2e2195bca2d4ab1878a7a84"), - CHTRoot: common.HexToHash("0x522ae1f334bfa36033b2315d0b9954052780700b69448ecea8d5877e0f7ee477"), - BloomRoot: common.HexToHash("0x4093fd53b0d2cc50181dca353fe66f03ae113e7cb65f869a4dfb5905de6a0493"), + SectionIndex: 393, + SectionHead: common.HexToHash("0x04479087c89428c6ed0d4ff25642776f0c35747d8ecef90547fa3ce4ebec8606"), + CHTRoot: common.HexToHash("0xaa100968cebe48dba3a8f196f044db04113d5a938ff083838ce6f2c588d416ad"), + BloomRoot: common.HexToHash("0xb9108d510c4b50b60793feead27620781bc1c2164e072d8022201c4eb7c36ba0"), } // RopstenCheckpointOracle contains a set of configs for the Ropsten test network oracle. @@ -141,29 +147,32 @@ var ( // SepoliaChainConfig contains the chain parameters to run a node on the Sepolia test network. SepoliaChainConfig = &ChainConfig{ - ChainID: big.NewInt(11155111), - HomesteadBlock: big.NewInt(0), - DAOForkBlock: nil, - DAOForkSupport: true, - EIP150Block: big.NewInt(0), - EIP155Block: big.NewInt(0), - EIP158Block: big.NewInt(0), - ByzantiumBlock: big.NewInt(0), - ConstantinopleBlock: big.NewInt(0), - PetersburgBlock: big.NewInt(0), - IstanbulBlock: big.NewInt(0), - MuirGlacierBlock: big.NewInt(0), - BerlinBlock: big.NewInt(0), - LondonBlock: big.NewInt(0), - Ethash: new(EthashConfig), + ChainID: big.NewInt(11155111), + HomesteadBlock: big.NewInt(0), + DAOForkBlock: nil, + DAOForkSupport: true, + EIP150Block: big.NewInt(0), + EIP155Block: big.NewInt(0), + EIP158Block: big.NewInt(0), + ByzantiumBlock: big.NewInt(0), + ConstantinopleBlock: big.NewInt(0), + PetersburgBlock: big.NewInt(0), + IstanbulBlock: big.NewInt(0), + MuirGlacierBlock: big.NewInt(0), + BerlinBlock: big.NewInt(0), + LondonBlock: big.NewInt(0), + TerminalTotalDifficulty: big.NewInt(17_000_000_000_000_000), + TerminalTotalDifficultyPassed: true, + MergeNetsplitBlock: big.NewInt(1735371), + Ethash: new(EthashConfig), } // SepoliaTrustedCheckpoint contains the light client trusted checkpoint for the Sepolia test network. SepoliaTrustedCheckpoint = &TrustedCheckpoint{ - SectionIndex: 34, - SectionHead: common.HexToHash("0xe361400fcbc468d641e7bdd0b0946a3548e97c5d2703b124f04a3f1deccec244"), - CHTRoot: common.HexToHash("0xea6768fd288dce7d84f590884908ec39e4de78e6e1a38de5c5419b0f49a42f91"), - BloomRoot: common.HexToHash("0x06d32f35d5a611bfd0333ad44e39c619449824167d8ef2913edc48a8112be2cd"), + SectionIndex: 55, + SectionHead: common.HexToHash("0xb70ea113ab4db9d6e015c5b55d486713f60c40bda666121914a71ce3aec53a75"), + CHTRoot: common.HexToHash("0x206456d8847b66aaf427ed551f55e24cff90241bdb0a02583c761bf8164f78e4"), + BloomRoot: common.HexToHash("0x4369228d59a8fe285fee874c636531091e659b3b1294bb978eb159860a1cede2"), } // RinkebyChainConfig contains the chain parameters to run a node on the Rinkeby test network. @@ -192,10 +201,10 @@ var ( // RinkebyTrustedCheckpoint contains the light client trusted checkpoint for the Rinkeby test network. RinkebyTrustedCheckpoint = &TrustedCheckpoint{ - SectionIndex: 326, - SectionHead: common.HexToHash("0x941a41a153b0e36cb15d9d193d1d0f9715bdb2435efd1c95119b64168667ce00"), - CHTRoot: common.HexToHash("0xe2331e00d579cf4093091dee35bef772e63c2341380c276041dc22563c8aba2e"), - BloomRoot: common.HexToHash("0x595206febcf118958c2bc1218ea71d01fd04b8f97ad71813df4be0af5b36b0e5"), + SectionIndex: 344, + SectionHead: common.HexToHash("0x06bb973aecce633df8cda532ff75b9d0b38c16de2545f52eaf745f858d0fe616"), + CHTRoot: common.HexToHash("0xf1c80b9270ef9fb7907362bca006f8349f0c38d45b83167b57638f54211c6aca"), + BloomRoot: common.HexToHash("0xd72187253f49bce9d471f5e0ddf2b5008ba695d7a1be1192d52fb4d8b01970c6"), } // RinkebyCheckpointOracle contains a set of configs for the Rinkeby test network oracle. @@ -212,21 +221,23 @@ var ( // GoerliChainConfig contains the chain parameters to run a node on the Görli test network. GoerliChainConfig = &ChainConfig{ - ChainID: big.NewInt(5), - HomesteadBlock: big.NewInt(0), - DAOForkBlock: nil, - DAOForkSupport: true, - EIP150Block: big.NewInt(0), - EIP155Block: big.NewInt(0), - EIP158Block: big.NewInt(0), - ByzantiumBlock: big.NewInt(0), - ConstantinopleBlock: big.NewInt(0), - PetersburgBlock: big.NewInt(0), - IstanbulBlock: big.NewInt(1_561_651), - MuirGlacierBlock: nil, - BerlinBlock: big.NewInt(4_460_644), - LondonBlock: big.NewInt(5_062_605), - ArrowGlacierBlock: nil, + ChainID: big.NewInt(5), + HomesteadBlock: big.NewInt(0), + DAOForkBlock: nil, + DAOForkSupport: true, + EIP150Block: big.NewInt(0), + EIP155Block: big.NewInt(0), + EIP158Block: big.NewInt(0), + ByzantiumBlock: big.NewInt(0), + ConstantinopleBlock: big.NewInt(0), + PetersburgBlock: big.NewInt(0), + IstanbulBlock: big.NewInt(1_561_651), + MuirGlacierBlock: nil, + BerlinBlock: big.NewInt(4_460_644), + LondonBlock: big.NewInt(5_062_605), + ArrowGlacierBlock: nil, + TerminalTotalDifficulty: big.NewInt(10_790_000), + TerminalTotalDifficultyPassed: true, Clique: &CliqueConfig{ Period: 15, Epoch: 30000, @@ -235,10 +246,10 @@ var ( // GoerliTrustedCheckpoint contains the light client trusted checkpoint for the Görli test network. GoerliTrustedCheckpoint = &TrustedCheckpoint{ - SectionIndex: 210, - SectionHead: common.HexToHash("0xbb11eaf551a6c06f74a6c7bbfe1699cbf64b8f248b64691da916dd443176db2f"), - CHTRoot: common.HexToHash("0x9934ae326d00d9c7de2e074c0e51689efb7fa7fcba18929ff4279c27259c45e6"), - BloomRoot: common.HexToHash("0x7fe3bd4fd45194aa8a5cfe5ac590edff1f870d3d98d3c310494e7f67613a87ff"), + SectionIndex: 229, + SectionHead: common.HexToHash("0xc5a7b57cb4af7b3d4cc251ac5f29acaac94e7464365358e7ad26129083b7729a"), + CHTRoot: common.HexToHash("0x54c0d5c756d9c48eda26ea13c2a49c2e31f1cb7dfb01514ddc49f3d24272c77e"), + BloomRoot: common.HexToHash("0xd681970a496f6187d089f8c8665a3587b5a78212d79b6ceef97c0dabd0188e56"), } // GoerliCheckpointOracle contains a set of configs for the Goerli test network oracle. @@ -259,19 +270,29 @@ var ( // // This configuration is intentionally not using keyed fields to force anyone // adding flags to the config to also have to set these fields. - AllEthashProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, new(EthashConfig), nil} + AllEthashProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, nil, nil, false, new(EthashConfig), nil} // AllCliqueProtocolChanges contains every protocol change (EIPs) introduced // and accepted by the Ethereum core developers into the Clique consensus. // // This configuration is intentionally not using keyed fields to force anyone // adding flags to the config to also have to set these fields. - AllCliqueProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, nil, nil, &CliqueConfig{Period: 0, Epoch: 30000}} + AllCliqueProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, nil, nil, nil, nil, false, nil, &CliqueConfig{Period: 0, Epoch: 30000}} - TestChainConfig = &ChainConfig{big.NewInt(1), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, new(EthashConfig), nil} - TestRules = TestChainConfig.Rules(new(big.Int), false) + TestChainConfig = &ChainConfig{big.NewInt(1), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, nil, nil, false, new(EthashConfig), nil} + NonActivatedConfig = &ChainConfig{big.NewInt(1), nil, nil, false, nil, common.Hash{}, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, false, new(EthashConfig), nil} + TestRules = TestChainConfig.Rules(new(big.Int), false) ) +// NetworkNames are user friendly names to use in the chain spec banner. +var NetworkNames = map[string]string{ + MainnetChainConfig.ChainID.String(): "mainnet", + RopstenChainConfig.ChainID.String(): "ropsten", + RinkebyChainConfig.ChainID.String(): "rinkeby", + GoerliChainConfig.ChainID.String(): "goerli", + SepoliaChainConfig.ChainID.String(): "sepolia", +} + // TrustedCheckpoint represents a set of post-processed trie roots (CHT and // BloomTrie) associated with the appropriate section index and head hash. It is // used to start light syncing from this checkpoint and avoid downloading the @@ -348,12 +369,20 @@ type ChainConfig struct { BerlinBlock *big.Int `json:"berlinBlock,omitempty"` // Berlin switch block (nil = no fork, 0 = already on berlin) LondonBlock *big.Int `json:"londonBlock,omitempty"` // London switch block (nil = no fork, 0 = already on london) ArrowGlacierBlock *big.Int `json:"arrowGlacierBlock,omitempty"` // Eip-4345 (bomb delay) switch block (nil = no fork, 0 = already activated) - MergeForkBlock *big.Int `json:"mergeForkBlock,omitempty"` // EIP-3675 (TheMerge) switch block (nil = no fork, 0 = already in merge proceedings) + GrayGlacierBlock *big.Int `json:"grayGlacierBlock,omitempty"` // Eip-5133 (bomb delay) switch block (nil = no fork, 0 = already activated) + MergeNetsplitBlock *big.Int `json:"mergeNetsplitBlock,omitempty"` // Virtual fork after The Merge to use as a network splitter + ShanghaiBlock *big.Int `json:"shanghaiBlock,omitempty"` // Shanghai switch block (nil = no fork, 0 = already on shanghai) + CancunBlock *big.Int `json:"cancunBlock,omitempty"` // Cancun switch block (nil = no fork, 0 = already on cancun) // TerminalTotalDifficulty is the amount of total difficulty reached by // the network that triggers the consensus upgrade. TerminalTotalDifficulty *big.Int `json:"terminalTotalDifficulty,omitempty"` + // TerminalTotalDifficultyPassed is a flag specifying that the network already + // passed the terminal total difficulty. Its purpose is to disable legacy sync + // even without having seen the TTD locally (safer long term). + TerminalTotalDifficultyPassed bool `json:"terminalTotalDifficultyPassed,omitempty"` + // Various consensus engines Ethash *EthashConfig `json:"ethash,omitempty"` Clique *CliqueConfig `json:"clique,omitempty"` @@ -378,37 +407,84 @@ func (c *CliqueConfig) String() string { return "clique" } -// String implements the fmt.Stringer interface. -func (c *ChainConfig) String() string { - var engine interface{} +// Description returns a human-readable description of ChainConfig. +func (c *ChainConfig) Description() string { + var banner string + + // Create some basinc network config output + network := NetworkNames[c.ChainID.String()] + if network == "" { + network = "unknown" + } + banner += fmt.Sprintf("Chain ID: %v (%s)\n", c.ChainID, network) switch { case c.Ethash != nil: - engine = c.Ethash + if c.TerminalTotalDifficulty == nil { + banner += "Consensus: Ethash (proof-of-work)\n" + } else if !c.TerminalTotalDifficultyPassed { + banner += "Consensus: Beacon (proof-of-stake), merging from Ethash (proof-of-work)\n" + } else { + banner += "Consensus: Beacon (proof-of-stake), merged from Ethash (proof-of-work)\n" + } case c.Clique != nil: - engine = c.Clique + if c.TerminalTotalDifficulty == nil { + banner += "Consensus: Clique (proof-of-authority)\n" + } else if !c.TerminalTotalDifficultyPassed { + banner += "Consensus: Beacon (proof-of-stake), merging from Clique (proof-of-authority)\n" + } else { + banner += "Consensus: Beacon (proof-of-stake), merged from Clique (proof-of-authority)\n" + } default: - engine = "unknown" - } - return fmt.Sprintf("{ChainID: %v Homestead: %v DAO: %v DAOSupport: %v EIP150: %v EIP155: %v EIP158: %v Byzantium: %v Constantinople: %v Petersburg: %v Istanbul: %v, Muir Glacier: %v, Berlin: %v, London: %v, Arrow Glacier: %v, MergeFork: %v, Terminal TD: %v, Engine: %v}", - c.ChainID, - c.HomesteadBlock, - c.DAOForkBlock, - c.DAOForkSupport, - c.EIP150Block, - c.EIP155Block, - c.EIP158Block, - c.ByzantiumBlock, - c.ConstantinopleBlock, - c.PetersburgBlock, - c.IstanbulBlock, - c.MuirGlacierBlock, - c.BerlinBlock, - c.LondonBlock, - c.ArrowGlacierBlock, - c.MergeForkBlock, - c.TerminalTotalDifficulty, - engine, - ) + banner += "Consensus: unknown\n" + } + banner += "\n" + + // Create a list of forks with a short description of them. Forks that only + // makes sense for mainnet should be optional at printing to avoid bloating + // the output for testnets and private networks. + banner += "Pre-Merge hard forks:\n" + banner += fmt.Sprintf(" - Homestead: %-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/homestead.md)\n", c.HomesteadBlock) + if c.DAOForkBlock != nil { + banner += fmt.Sprintf(" - DAO Fork: %-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/dao-fork.md)\n", c.DAOForkBlock) + } + banner += fmt.Sprintf(" - Tangerine Whistle (EIP 150): %-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/tangerine-whistle.md)\n", c.EIP150Block) + banner += fmt.Sprintf(" - Spurious Dragon/1 (EIP 155): %-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/spurious-dragon.md)\n", c.EIP155Block) + banner += fmt.Sprintf(" - Spurious Dragon/2 (EIP 158): %-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/spurious-dragon.md)\n", c.EIP155Block) + banner += fmt.Sprintf(" - Byzantium: %-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/byzantium.md)\n", c.ByzantiumBlock) + banner += fmt.Sprintf(" - Constantinople: %-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/constantinople.md)\n", c.ConstantinopleBlock) + banner += fmt.Sprintf(" - Petersburg: %-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/petersburg.md)\n", c.PetersburgBlock) + banner += fmt.Sprintf(" - Istanbul: %-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/istanbul.md)\n", c.IstanbulBlock) + if c.MuirGlacierBlock != nil { + banner += fmt.Sprintf(" - Muir Glacier: %-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/muir-glacier.md)\n", c.MuirGlacierBlock) + } + banner += fmt.Sprintf(" - Berlin: %-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/berlin.md)\n", c.BerlinBlock) + banner += fmt.Sprintf(" - London: %-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/london.md)\n", c.LondonBlock) + if c.ArrowGlacierBlock != nil { + banner += fmt.Sprintf(" - Arrow Glacier: %-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/arrow-glacier.md)\n", c.ArrowGlacierBlock) + } + if c.GrayGlacierBlock != nil { + banner += fmt.Sprintf(" - Gray Glacier: %-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/gray-glacier.md)\n", c.GrayGlacierBlock) + } + if c.ShanghaiBlock != nil { + banner += fmt.Sprintf(" - Shanghai: %-8v (https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/shanghai.md)\n", c.ShanghaiBlock) + } + if c.CancunBlock != nil { + banner += fmt.Sprintf(" - Cancun: %-8v\n", c.CancunBlock) + } + banner += "\n" + + // Add a special section for the merge as it's non-obvious + if c.TerminalTotalDifficulty == nil { + banner += "The Merge is not yet available for this network!\n" + banner += " - Hard-fork specification: https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/paris.md" + } else { + banner += "Merge configured:\n" + banner += " - Hard-fork specification: https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/paris.md\n" + banner += fmt.Sprintf(" - Network known to be merged: %v\n", c.TerminalTotalDifficultyPassed) + banner += fmt.Sprintf(" - Total terminal difficulty: %v\n", c.TerminalTotalDifficulty) + banner += fmt.Sprintf(" - Merge netsplit block: %-8v", c.MergeNetsplitBlock) + } + return banner } // IsHomestead returns whether num is either equal to the homestead block or greater. @@ -478,6 +554,11 @@ func (c *ChainConfig) IsArrowGlacier(num *big.Int) bool { return isForked(c.ArrowGlacierBlock, num) } +// IsGrayGlacier returns whether num is either equal to the Gray Glacier (EIP-5133) fork block or greater. +func (c *ChainConfig) IsGrayGlacier(num *big.Int) bool { + return isForked(c.GrayGlacierBlock, num) +} + // IsTerminalPoWBlock returns whether the given block is the last block of PoW stage. func (c *ChainConfig) IsTerminalPoWBlock(parentTotalDiff *big.Int, totalDiff *big.Int) bool { if c.TerminalTotalDifficulty == nil { @@ -486,6 +567,16 @@ func (c *ChainConfig) IsTerminalPoWBlock(parentTotalDiff *big.Int, totalDiff *bi return parentTotalDiff.Cmp(c.TerminalTotalDifficulty) < 0 && totalDiff.Cmp(c.TerminalTotalDifficulty) >= 0 } +// IsShanghai returns whether num is either equal to the Shanghai fork block or greater. +func (c *ChainConfig) IsShanghai(num *big.Int) bool { + return isForked(c.ShanghaiBlock, num) +} + +// IsCancun returns whether num is either equal to the Cancun fork block or greater. +func (c *ChainConfig) IsCancun(num *big.Int) bool { + return isForked(c.CancunBlock, num) +} + // CheckCompatible checks whether scheduled fork transitions have been imported // with a mismatching chain configuration. func (c *ChainConfig) CheckCompatible(newcfg *ChainConfig, height uint64) *ConfigCompatError { @@ -527,7 +618,10 @@ func (c *ChainConfig) CheckConfigForkOrder() error { {name: "berlinBlock", block: c.BerlinBlock}, {name: "londonBlock", block: c.LondonBlock}, {name: "arrowGlacierBlock", block: c.ArrowGlacierBlock, optional: true}, - {name: "mergeStartBlock", block: c.MergeForkBlock, optional: true}, + {name: "grayGlacierBlock", block: c.GrayGlacierBlock, optional: true}, + {name: "mergeNetsplitBlock", block: c.MergeNetsplitBlock, optional: true}, + {name: "shanghaiBlock", block: c.ShanghaiBlock, optional: true}, + {name: "cancunBlock", block: c.CancunBlock, optional: true}, } { if lastFork.name != "" { // Next one must be higher number @@ -600,8 +694,17 @@ func (c *ChainConfig) checkCompatible(newcfg *ChainConfig, head *big.Int) *Confi if isForkIncompatible(c.ArrowGlacierBlock, newcfg.ArrowGlacierBlock, head) { return newCompatError("Arrow Glacier fork block", c.ArrowGlacierBlock, newcfg.ArrowGlacierBlock) } - if isForkIncompatible(c.MergeForkBlock, newcfg.MergeForkBlock, head) { - return newCompatError("Merge Start fork block", c.MergeForkBlock, newcfg.MergeForkBlock) + if isForkIncompatible(c.GrayGlacierBlock, newcfg.GrayGlacierBlock, head) { + return newCompatError("Gray Glacier fork block", c.GrayGlacierBlock, newcfg.GrayGlacierBlock) + } + if isForkIncompatible(c.MergeNetsplitBlock, newcfg.MergeNetsplitBlock, head) { + return newCompatError("Merge netsplit fork block", c.MergeNetsplitBlock, newcfg.MergeNetsplitBlock) + } + if isForkIncompatible(c.ShanghaiBlock, newcfg.ShanghaiBlock, head) { + return newCompatError("Shanghai fork block", c.ShanghaiBlock, newcfg.ShanghaiBlock) + } + if isForkIncompatible(c.CancunBlock, newcfg.CancunBlock, head) { + return newCompatError("Cancun fork block", c.CancunBlock, newcfg.CancunBlock) } return nil } @@ -671,7 +774,7 @@ type Rules struct { IsHomestead, IsEIP150, IsEIP155, IsEIP158 bool IsByzantium, IsConstantinople, IsPetersburg, IsIstanbul bool IsBerlin, IsLondon bool - IsMerge bool + IsMerge, IsShanghai, isCancun bool } // Rules ensures c's ChainID is not nil. @@ -693,5 +796,7 @@ func (c *ChainConfig) Rules(num *big.Int, isMerge bool) Rules { IsBerlin: c.IsBerlin(num), IsLondon: c.IsLondon(num), IsMerge: isMerge, + IsShanghai: c.IsShanghai(num), + isCancun: c.IsCancun(num), } } diff --git a/params/denomination.go b/params/denomination.go index fb4da7f4125a2..bcedd271e0e21 100644 --- a/params/denomination.go +++ b/params/denomination.go @@ -19,8 +19,7 @@ package params // These are the multipliers for ether denominations. // Example: To get the wei value of an amount in 'gwei', use // -// new(big.Int).Mul(value, big.NewInt(params.GWei)) -// +// new(big.Int).Mul(value, big.NewInt(params.GWei)) const ( Wei = 1 GWei = 1e9 diff --git a/params/version.go b/params/version.go index 8c8b6295a8f07..3172415393257 100644 --- a/params/version.go +++ b/params/version.go @@ -22,8 +22,8 @@ import ( const ( VersionMajor = 1 // Major version component of the current release - VersionMinor = 10 // Minor version component of the current release - VersionPatch = 19 // Patch version component of the current release + VersionMinor = 11 // Minor version component of the current release + VersionPatch = 0 // Patch version component of the current release VersionMeta = "unstable" // Version metadata to append to the version string ) @@ -41,9 +41,9 @@ var VersionWithMeta = func() string { return v }() -// ArchiveVersion holds the textual version string used for Geth archives. -// e.g. "1.8.11-dea1ce05" for stable releases, or -// "1.8.13-unstable-21c059b6" for unstable releases +// ArchiveVersion holds the textual version string used for Geth archives. e.g. +// "1.8.11-dea1ce05" for stable releases, or "1.8.13-unstable-21c059b6" for unstable +// releases. func ArchiveVersion(gitCommit string) string { vsn := Version if VersionMeta != "stable" { diff --git a/rlp/decode.go b/rlp/decode.go index 9214dbfb37201..c9b2652414558 100644 --- a/rlp/decode.go +++ b/rlp/decode.go @@ -76,7 +76,7 @@ type Decoder interface { // Note that Decode does not set an input limit for all readers and may be vulnerable to // panics cause by huge value sizes. If you need an input limit, use // -// NewStream(r, limit).Decode(val) +// NewStream(r, limit).Decode(val) func Decode(r io.Reader, val interface{}) error { stream := streamPool.Get().(*Stream) defer streamPool.Put(stream) diff --git a/rlp/decode_test.go b/rlp/decode_test.go index 46aa68cea3d73..00722f847bbbc 100644 --- a/rlp/decode_test.go +++ b/rlp/decode_test.go @@ -452,7 +452,7 @@ type ignoredField struct { var ( veryBigInt = new(big.Int).Add( - big.NewInt(0).Lsh(big.NewInt(0xFFFFFFFFFFFFFF), 16), + new(big.Int).Lsh(big.NewInt(0xFFFFFFFFFFFFFF), 16), big.NewInt(0xFFFF), ) veryVeryBigInt = new(big.Int).Exp(veryBigInt, big.NewInt(8), nil) @@ -1043,7 +1043,6 @@ func TestInvalidOptionalField(t *testing.T) { t.Errorf("wrong error for %T: %v", test.v, err.Error()) } } - } func ExampleDecode() { diff --git a/rlp/doc.go b/rlp/doc.go index e4404c978da7a..eeeee9a43a0cd 100644 --- a/rlp/doc.go +++ b/rlp/doc.go @@ -27,8 +27,7 @@ value zero equivalent to the empty string). RLP values are distinguished by a type tag. The type tag precedes the value in the input stream and defines the size and kind of the bytes that follow. - -Encoding Rules +# Encoding Rules Package rlp uses reflection and encodes RLP based on the Go type of the value. @@ -58,8 +57,7 @@ An interface value encodes as the value contained in the interface. Floating point numbers, maps, channels and functions are not supported. - -Decoding Rules +# Decoding Rules Decoding uses the following type-dependent rules: @@ -93,30 +91,29 @@ or one (true). To decode into an interface value, one of these types is stored in the value: - []interface{}, for RLP lists - []byte, for RLP strings + []interface{}, for RLP lists + []byte, for RLP strings Non-empty interface types are not supported when decoding. Signed integers, floating point numbers, maps, channels and functions cannot be decoded into. - -Struct Tags +# Struct Tags As with other encoding packages, the "-" tag ignores fields. - type StructWithIgnoredField struct{ - Ignored uint `rlp:"-"` - Field uint - } + type StructWithIgnoredField struct{ + Ignored uint `rlp:"-"` + Field uint + } Go struct values encode/decode as RLP lists. There are two ways of influencing the mapping of fields to list elements. The "tail" tag, which may only be used on the last exported struct field, allows slurping up any excess list elements into a slice. - type StructWithTail struct{ - Field uint - Tail []string `rlp:"tail"` - } + type StructWithTail struct{ + Field uint + Tail []string `rlp:"tail"` + } The "optional" tag says that the field may be omitted if it is zero-valued. If this tag is used on a struct field, all subsequent public fields must also be declared optional. @@ -128,11 +125,11 @@ When decoding into a struct, optional fields may be omitted from the end of the list. For the example below, this means input lists of one, two, or three elements are accepted. - type StructWithOptionalFields struct{ - Required uint - Optional1 uint `rlp:"optional"` - Optional2 uint `rlp:"optional"` - } + type StructWithOptionalFields struct{ + Required uint + Optional1 uint `rlp:"optional"` + Optional2 uint `rlp:"optional"` + } The "nil", "nilList" and "nilString" tags apply to pointer-typed fields only, and change the decoding rules for the field type. For regular pointer fields without the "nil" tag, @@ -140,9 +137,9 @@ input values must always match the required input length exactly and the decoder produce nil values. When the "nil" tag is set, input values of size zero decode as a nil pointer. This is especially useful for recursive types. - type StructWithNilField struct { - Field *[3]byte `rlp:"nil"` - } + type StructWithNilField struct { + Field *[3]byte `rlp:"nil"` + } In the example above, Field allows two possible input sizes. For input 0xC180 (a list containing an empty string) Field is set to nil after decoding. For input 0xC483000000 (a diff --git a/rlp/encode_test.go b/rlp/encode_test.go index 78392906b557e..58ddc0d120f0f 100644 --- a/rlp/encode_test.go +++ b/rlp/encode_test.go @@ -119,15 +119,15 @@ var encTests = []encTest{ {val: big.NewInt(0xFFFFFFFFFFFF), output: "86FFFFFFFFFFFF"}, {val: big.NewInt(0xFFFFFFFFFFFFFF), output: "87FFFFFFFFFFFFFF"}, { - val: big.NewInt(0).SetBytes(unhex("102030405060708090A0B0C0D0E0F2")), + val: new(big.Int).SetBytes(unhex("102030405060708090A0B0C0D0E0F2")), output: "8F102030405060708090A0B0C0D0E0F2", }, { - val: big.NewInt(0).SetBytes(unhex("0100020003000400050006000700080009000A000B000C000D000E01")), + val: new(big.Int).SetBytes(unhex("0100020003000400050006000700080009000A000B000C000D000E01")), output: "9C0100020003000400050006000700080009000A000B000C000D000E01", }, { - val: big.NewInt(0).SetBytes(unhex("010000000000000000000000000000000000000000000000000000000000000000")), + val: new(big.Int).SetBytes(unhex("010000000000000000000000000000000000000000000000000000000000000000")), output: "A1010000000000000000000000000000000000000000000000000000000000000000", }, { diff --git a/rlp/iterator.go b/rlp/iterator.go index 353ef09fbdf25..6be574572e611 100644 --- a/rlp/iterator.go +++ b/rlp/iterator.go @@ -36,7 +36,6 @@ func NewListIterator(data RawValue) (*listIterator, error) { data: data[t : t+c], } return it, nil - } // Next forwards the iterator one step, returns true if it was not at end yet diff --git a/rlp/rlpgen/main.go b/rlp/rlpgen/main.go index 6258fdb47a563..25d4393cc6561 100644 --- a/rlp/rlpgen/main.go +++ b/rlp/rlpgen/main.go @@ -51,7 +51,7 @@ func main() { } if *output == "-" { os.Stdout.Write(code) - } else if err := os.WriteFile(*output, code, 0644); err != nil { + } else if err := os.WriteFile(*output, code, 0600); err != nil { fatal(err) } } @@ -106,7 +106,7 @@ func (cfg *Config) process() (code []byte, err error) { // Find the type and generate. typ, err := lookupStructType(pkg.Scope(), cfg.Type) if err != nil { - return nil, fmt.Errorf("can't find %s in %s: %v", typ, pkg, err) + return nil, fmt.Errorf("can't find %s in %s: %v", cfg.Type, pkg, err) } code, err = bctx.generate(typ, cfg.GenerateEncoder, cfg.GenerateDecoder) if err != nil { diff --git a/rpc/client.go b/rpc/client.go index d3ce0297754c9..8288f976ebeb2 100644 --- a/rpc/client.go +++ b/rpc/client.go @@ -22,6 +22,7 @@ import ( "errors" "fmt" "net/url" + "os" "reflect" "strconv" "sync/atomic" @@ -99,7 +100,7 @@ type Client struct { reqTimeout chan *requestOp // removes response IDs when call timeout expires } -type reconnectFunc func(ctx context.Context) (ServerCodec, error) +type reconnectFunc func(context.Context) (ServerCodec, error) type clientContextKey struct{} @@ -153,14 +154,16 @@ func (op *requestOp) wait(ctx context.Context, c *Client) (*jsonrpcMessage, erro // // The currently supported URL schemes are "http", "https", "ws" and "wss". If rawurl is a // file name with no URL scheme, a local socket connection is established using UNIX -// domain sockets on supported platforms and named pipes on Windows. If you want to -// configure transport options, use DialHTTP, DialWebsocket or DialIPC instead. +// domain sockets on supported platforms and named pipes on Windows. +// +// If you want to further configure the transport, use DialOptions instead of this +// function. // // For websocket connections, the origin is set to the local host name. // -// The client reconnects automatically if the connection is lost. +// The client reconnects automatically when the connection is lost. func Dial(rawurl string) (*Client, error) { - return DialContext(context.Background(), rawurl) + return DialOptions(context.Background(), rawurl) } // DialContext creates a new RPC client, just like Dial. @@ -168,22 +171,46 @@ func Dial(rawurl string) (*Client, error) { // The context is used to cancel or time out the initial connection establishment. It does // not affect subsequent interactions with the client. func DialContext(ctx context.Context, rawurl string) (*Client, error) { + return DialOptions(ctx, rawurl) +} + +// DialOptions creates a new RPC client for the given URL. You can supply any of the +// pre-defined client options to configure the underlying transport. +// +// The context is used to cancel or time out the initial connection establishment. It does +// not affect subsequent interactions with the client. +// +// The client reconnects automatically when the connection is lost. +func DialOptions(ctx context.Context, rawurl string, options ...ClientOption) (*Client, error) { u, err := url.Parse(rawurl) if err != nil { return nil, err } + + cfg := new(clientConfig) + for _, opt := range options { + opt.applyOption(cfg) + } + + var reconnect reconnectFunc switch u.Scheme { case "http", "https": - return DialHTTP(rawurl) + reconnect = newClientTransportHTTP(rawurl, cfg) case "ws", "wss": - return DialWebsocket(ctx, rawurl, "") + rc, err := newClientTransportWS(rawurl, cfg) + if err != nil { + return nil, err + } + reconnect = rc case "stdio": - return DialStdIO(ctx) + reconnect = newClientTransportIO(os.Stdin, os.Stdout) case "": - return DialIPC(ctx, rawurl) + reconnect = newClientTransportIPC(rawurl) default: return nil, fmt.Errorf("no known transport for URL scheme %q", u.Scheme) } + + return newClient(ctx, reconnect) } // ClientFromContext retrieves the client from the context, if any. This can be used to perform diff --git a/rpc/client_opt.go b/rpc/client_opt.go new file mode 100644 index 0000000000000..5ad7c22b3ce76 --- /dev/null +++ b/rpc/client_opt.go @@ -0,0 +1,106 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rpc + +import ( + "net/http" + + "github.com/gorilla/websocket" +) + +// ClientOption is a configuration option for the RPC client. +type ClientOption interface { + applyOption(*clientConfig) +} + +type clientConfig struct { + httpClient *http.Client + httpHeaders http.Header + httpAuth HTTPAuth + + wsDialer *websocket.Dialer +} + +func (cfg *clientConfig) initHeaders() { + if cfg.httpHeaders == nil { + cfg.httpHeaders = make(http.Header) + } +} + +func (cfg *clientConfig) setHeader(key, value string) { + cfg.initHeaders() + cfg.httpHeaders.Set(key, value) +} + +type optionFunc func(*clientConfig) + +func (fn optionFunc) applyOption(opt *clientConfig) { + fn(opt) +} + +// WithWebsocketDialer configures the websocket.Dialer used by the RPC client. +func WithWebsocketDialer(dialer websocket.Dialer) ClientOption { + return optionFunc(func(cfg *clientConfig) { + cfg.wsDialer = &dialer + }) +} + +// WithHeader configures HTTP headers set by the RPC client. Headers set using this option +// will be used for both HTTP and WebSocket connections. +func WithHeader(key, value string) ClientOption { + return optionFunc(func(cfg *clientConfig) { + cfg.initHeaders() + cfg.httpHeaders.Set(key, value) + }) +} + +// WithHeaders configures HTTP headers set by the RPC client. Headers set using this +// option will be used for both HTTP and WebSocket connections. +func WithHeaders(headers http.Header) ClientOption { + return optionFunc(func(cfg *clientConfig) { + cfg.initHeaders() + for k, vs := range headers { + cfg.httpHeaders[k] = vs + } + }) +} + +// WithHTTPClient configures the http.Client used by the RPC client. +func WithHTTPClient(c *http.Client) ClientOption { + return optionFunc(func(cfg *clientConfig) { + cfg.httpClient = c + }) +} + +// WithHTTPAuth configures HTTP request authentication. The given provider will be called +// whenever a request is made. Note that only one authentication provider can be active at +// any time. +func WithHTTPAuth(a HTTPAuth) ClientOption { + if a == nil { + panic("nil auth") + } + return optionFunc(func(cfg *clientConfig) { + cfg.httpAuth = a + }) +} + +// A HTTPAuth function is called by the client whenever a HTTP request is sent. +// The function must be safe for concurrent use. +// +// Usually, HTTPAuth functions will call h.Set("authorization", "...") to add +// auth information to the request. +type HTTPAuth func(h http.Header) error diff --git a/rpc/client_opt_test.go b/rpc/client_opt_test.go new file mode 100644 index 0000000000000..d7cc2572a776f --- /dev/null +++ b/rpc/client_opt_test.go @@ -0,0 +1,25 @@ +package rpc_test + +import ( + "context" + "net/http" + "time" + + "github.com/ethereum/go-ethereum/rpc" +) + +// This example configures a HTTP-based RPC client with two options - one setting the +// overall request timeout, the other adding a custom HTTP header to all requests. +func ExampleDialOptions() { + tokenHeader := rpc.WithHeader("x-token", "foo") + httpClient := rpc.WithHTTPClient(&http.Client{ + Timeout: 10 * time.Second, + }) + + ctx := context.Background() + c, err := rpc.DialOptions(ctx, "http://rpc.example.com", httpClient, tokenHeader) + if err != nil { + panic(err) + } + c.Close() +} diff --git a/rpc/client_test.go b/rpc/client_test.go index fa6010bb199c1..51df76f7fe443 100644 --- a/rpc/client_test.go +++ b/rpc/client_test.go @@ -82,11 +82,15 @@ func TestClientErrorData(t *testing.T) { } // Check code. + // The method handler returns an error value which implements the rpc.Error + // interface, i.e. it has a custom error code. The server returns this error code. + expectedCode := testError{}.ErrorCode() if e, ok := err.(Error); !ok { t.Fatalf("client did not return rpc.Error, got %#v", e) - } else if e.ErrorCode() != (testError{}.ErrorCode()) { - t.Fatalf("wrong error code %d, want %d", e.ErrorCode(), testError{}.ErrorCode()) + } else if e.ErrorCode() != expectedCode { + t.Fatalf("wrong error code %d, want %d", e.ErrorCode(), expectedCode) } + // Check data. if e, ok := err.(DataError); !ok { t.Fatalf("client did not return rpc.DataError, got %#v", e) @@ -615,10 +619,10 @@ func TestClientReconnect(t *testing.T) { // Start a server and corresponding client. s1, l1 := startServer("127.0.0.1:0") client, err := DialContext(ctx, "ws://"+l1.Addr().String()) - defer client.Close() if err != nil { t.Fatal("can't dial", err) } + defer client.Close() // Perform a call. This should work because the server is up. var resp echoResult diff --git a/rpc/doc.go b/rpc/doc.go index e0a6324675e68..7c87793dcab6d 100644 --- a/rpc/doc.go +++ b/rpc/doc.go @@ -15,7 +15,6 @@ // along with the go-ethereum library. If not, see . /* - Package rpc implements bi-directional JSON-RPC 2.0 on multiple transports. It provides access to the exported methods of an object across a network or other I/O @@ -23,16 +22,16 @@ connection. After creating a server or client instance, objects can be registere them visible as 'services'. Exported methods that follow specific conventions can be called remotely. It also has support for the publish/subscribe pattern. -RPC Methods +# RPC Methods Methods that satisfy the following criteria are made available for remote access: - - method must be exported - - method returns 0, 1 (response or error) or 2 (response and error) values + - method must be exported + - method returns 0, 1 (response or error) or 2 (response and error) values An example method: - func (s *CalcService) Add(a, b int) (int, error) + func (s *CalcService) Add(a, b int) (int, error) When the returned error isn't nil the returned integer is ignored and the error is sent back to the client. Otherwise the returned integer is sent back to the client. @@ -41,7 +40,7 @@ Optional arguments are supported by accepting pointer values as arguments. E.g. to do the addition in an optional finite field we can accept a mod argument as pointer value. - func (s *CalcService) Add(a, b int, mod *int) (int, error) + func (s *CalcService) Add(a, b int, mod *int) (int, error) This RPC method can be called with 2 integers and a null value as third argument. In that case the mod argument will be nil. Or it can be called with 3 integers, in that case mod @@ -56,40 +55,40 @@ to the client out of order. An example server which uses the JSON codec: - type CalculatorService struct {} + type CalculatorService struct {} - func (s *CalculatorService) Add(a, b int) int { - return a + b - } + func (s *CalculatorService) Add(a, b int) int { + return a + b + } - func (s *CalculatorService) Div(a, b int) (int, error) { - if b == 0 { - return 0, errors.New("divide by zero") - } - return a/b, nil - } + func (s *CalculatorService) Div(a, b int) (int, error) { + if b == 0 { + return 0, errors.New("divide by zero") + } + return a/b, nil + } - calculator := new(CalculatorService) - server := NewServer() - server.RegisterName("calculator", calculator) - l, _ := net.ListenUnix("unix", &net.UnixAddr{Net: "unix", Name: "/tmp/calculator.sock"}) - server.ServeListener(l) + calculator := new(CalculatorService) + server := NewServer() + server.RegisterName("calculator", calculator) + l, _ := net.ListenUnix("unix", &net.UnixAddr{Net: "unix", Name: "/tmp/calculator.sock"}) + server.ServeListener(l) -Subscriptions +# Subscriptions The package also supports the publish subscribe pattern through the use of subscriptions. A method that is considered eligible for notifications must satisfy the following criteria: - - method must be exported - - first method argument type must be context.Context - - method must have return types (rpc.Subscription, error) + - method must be exported + - first method argument type must be context.Context + - method must have return types (rpc.Subscription, error) An example method: - func (s *BlockChainService) NewBlocks(ctx context.Context) (rpc.Subscription, error) { - ... - } + func (s *BlockChainService) NewBlocks(ctx context.Context) (rpc.Subscription, error) { + ... + } When the service containing the subscription method is registered to the server, for example under the "blockchain" namespace, a subscription is created by calling the @@ -101,7 +100,7 @@ the client and server. The server will close the connection for any write error. For more information about subscriptions, see https://github.com/ethereum/go-ethereum/wiki/RPC-PUB-SUB. -Reverse Calls +# Reverse Calls In any method handler, an instance of rpc.Client can be accessed through the ClientFromContext method. Using this client instance, server-to-client method calls can be diff --git a/rpc/errors.go b/rpc/errors.go index 4c06a745fbd8b..9a19e9fe67f59 100644 --- a/rpc/errors.go +++ b/rpc/errors.go @@ -54,9 +54,15 @@ var ( _ Error = new(invalidRequestError) _ Error = new(invalidMessageError) _ Error = new(invalidParamsError) + _ Error = new(internalServerError) ) -const defaultErrorCode = -32000 +const ( + errcodeDefault = -32000 + errcodeNotificationsUnsupported = -32001 + errcodePanic = -32603 + errcodeMarshalError = -32603 +) type methodNotFoundError struct{ method string } @@ -101,3 +107,13 @@ type invalidParamsError struct{ message string } func (e *invalidParamsError) ErrorCode() int { return -32602 } func (e *invalidParamsError) Error() string { return e.message } + +// internalServerError is used for server errors during request processing. +type internalServerError struct { + code int + message string +} + +func (e *internalServerError) ErrorCode() int { return e.code } + +func (e *internalServerError) Error() string { return e.message } diff --git a/rpc/handler.go b/rpc/handler.go index e8d1887c7d236..f3052e7eb8225 100644 --- a/rpc/handler.go +++ b/rpc/handler.go @@ -34,21 +34,20 @@ import ( // // The entry points for incoming messages are: // -// h.handleMsg(message) -// h.handleBatch(message) +// h.handleMsg(message) +// h.handleBatch(message) // // Outgoing calls use the requestOp struct. Register the request before sending it // on the connection: // -// op := &requestOp{ids: ...} -// h.addRequestOp(op) +// op := &requestOp{ids: ...} +// h.addRequestOp(op) // // Now send the request, then wait for the reply to be delivered through handleMsg: // -// if err := op.wait(...); err != nil { -// h.removeRequestOp(op) // timeout, etc. -// } -// +// if err := op.wait(...); err != nil { +// h.removeRequestOp(op) // timeout, etc. +// } type handler struct { reg *serviceRegistry unsubscribeCb *callback @@ -346,7 +345,7 @@ func (h *handler) handleCall(cp *callProc, msg *jsonrpcMessage) *jsonrpcMessage successfulRequestGauge.Inc(1) } rpcServingTimer.UpdateSince(start) - newRPCServingTimer(msg.Method, answer.Error == nil).UpdateSince(start) + updateServeTimeHistogram(msg.Method, answer.Error == nil, time.Since(start)) } return answer } @@ -354,7 +353,10 @@ func (h *handler) handleCall(cp *callProc, msg *jsonrpcMessage) *jsonrpcMessage // handleSubscribe processes *_subscribe method calls. func (h *handler) handleSubscribe(cp *callProc, msg *jsonrpcMessage) *jsonrpcMessage { if !h.allowSubscribe { - return msg.errorResponse(ErrNotificationsUnsupported) + return msg.errorResponse(&internalServerError{ + code: errcodeNotificationsUnsupported, + message: ErrNotificationsUnsupported.Error(), + }) } // Subscription method name is first argument. diff --git a/rpc/http.go b/rpc/http.go index 9f44649573493..8595959afb660 100644 --- a/rpc/http.go +++ b/rpc/http.go @@ -45,6 +45,7 @@ type httpConn struct { closeCh chan interface{} mu sync.Mutex // protects headers headers http.Header + auth HTTPAuth } // httpConn implements ServerCodec, but it is treated specially by Client @@ -87,6 +88,14 @@ type HTTPTimeouts struct { // ReadHeaderTimeout. It is valid to use them both. ReadTimeout time.Duration + // ReadHeaderTimeout is the amount of time allowed to read + // request headers. The connection's read deadline is reset + // after reading the headers and the Handler can decide what + // is considered too slow for the body. If ReadHeaderTimeout + // is zero, the value of ReadTimeout is used. If both are + // zero, there is no timeout. + ReadHeaderTimeout time.Duration + // WriteTimeout is the maximum duration before timing out // writes of the response. It is reset whenever a new // request's header is read. Like ReadTimeout, it does not @@ -103,13 +112,21 @@ type HTTPTimeouts struct { // DefaultHTTPTimeouts represents the default timeout values used if further // configuration is not provided. var DefaultHTTPTimeouts = HTTPTimeouts{ - ReadTimeout: 30 * time.Second, - WriteTimeout: 30 * time.Second, - IdleTimeout: 120 * time.Second, + ReadTimeout: 30 * time.Second, + ReadHeaderTimeout: 30 * time.Second, + WriteTimeout: 30 * time.Second, + IdleTimeout: 120 * time.Second, +} + +// DialHTTP creates a new RPC client that connects to an RPC server over HTTP. +func DialHTTP(endpoint string) (*Client, error) { + return DialHTTPWithClient(endpoint, new(http.Client)) } // DialHTTPWithClient creates a new RPC client that connects to an RPC server over HTTP // using the provided HTTP Client. +// +// Deprecated: use DialOptions and the WithHTTPClient option. func DialHTTPWithClient(endpoint string, client *http.Client) (*Client, error) { // Sanity check URL so we don't end up with a client that will fail every request. _, err := url.Parse(endpoint) @@ -117,24 +134,35 @@ func DialHTTPWithClient(endpoint string, client *http.Client) (*Client, error) { return nil, err } - initctx := context.Background() - headers := make(http.Header, 2) + var cfg clientConfig + fn := newClientTransportHTTP(endpoint, &cfg) + return newClient(context.Background(), fn) +} + +func newClientTransportHTTP(endpoint string, cfg *clientConfig) reconnectFunc { + headers := make(http.Header, 2+len(cfg.httpHeaders)) headers.Set("accept", contentType) headers.Set("content-type", contentType) - return newClient(initctx, func(context.Context) (ServerCodec, error) { - hc := &httpConn{ - client: client, - headers: headers, - url: endpoint, - closeCh: make(chan interface{}), - } - return hc, nil - }) -} + for key, values := range cfg.httpHeaders { + headers[key] = values + } -// DialHTTP creates a new RPC client that connects to an RPC server over HTTP. -func DialHTTP(endpoint string) (*Client, error) { - return DialHTTPWithClient(endpoint, new(http.Client)) + client := cfg.httpClient + if client == nil { + client = new(http.Client) + } + + hc := &httpConn{ + client: client, + headers: headers, + url: endpoint, + auth: cfg.httpAuth, + closeCh: make(chan interface{}), + } + + return func(ctx context.Context) (ServerCodec, error) { + return hc, nil + } } func (c *Client) sendHTTP(ctx context.Context, op *requestOp, msg interface{}) error { @@ -186,6 +214,11 @@ func (hc *httpConn) doRequest(ctx context.Context, msg interface{}) (io.ReadClos hc.mu.Lock() req.Header = hc.headers.Clone() hc.mu.Unlock() + if hc.auth != nil { + if err := hc.auth(req.Header); err != nil { + return nil, err + } + } // do request resp, err := hc.client.Do(req) diff --git a/rpc/ipc.go b/rpc/ipc.go index 07a211c6277c4..d9e0de62e877d 100644 --- a/rpc/ipc.go +++ b/rpc/ipc.go @@ -46,11 +46,15 @@ func (s *Server) ServeListener(l net.Listener) error { // The context is used for the initial connection establishment. It does not // affect subsequent interactions with the client. func DialIPC(ctx context.Context, endpoint string) (*Client, error) { - return newClient(ctx, func(ctx context.Context) (ServerCodec, error) { + return newClient(ctx, newClientTransportIPC(endpoint)) +} + +func newClientTransportIPC(endpoint string) reconnectFunc { + return func(ctx context.Context) (ServerCodec, error) { conn, err := newIPCConnection(ctx, endpoint) if err != nil { return nil, err } return NewCodec(conn), err - }) + } } diff --git a/rpc/json.go b/rpc/json.go index 6024f1e7dc9bf..1064939ff8b6c 100644 --- a/rpc/json.go +++ b/rpc/json.go @@ -58,21 +58,25 @@ type jsonrpcMessage struct { } func (msg *jsonrpcMessage) isNotification() bool { - return msg.ID == nil && msg.Method != "" + return msg.hasValidVersion() && msg.ID == nil && msg.Method != "" } func (msg *jsonrpcMessage) isCall() bool { - return msg.hasValidID() && msg.Method != "" + return msg.hasValidVersion() && msg.hasValidID() && msg.Method != "" } func (msg *jsonrpcMessage) isResponse() bool { - return msg.hasValidID() && msg.Method == "" && msg.Params == nil && (msg.Result != nil || msg.Error != nil) + return msg.hasValidVersion() && msg.hasValidID() && msg.Method == "" && msg.Params == nil && (msg.Result != nil || msg.Error != nil) } func (msg *jsonrpcMessage) hasValidID() bool { return len(msg.ID) > 0 && msg.ID[0] != '{' && msg.ID[0] != '[' } +func (msg *jsonrpcMessage) hasValidVersion() bool { + return msg.Version == vsn +} + func (msg *jsonrpcMessage) isSubscribe() bool { return strings.HasSuffix(msg.Method, subscribeMethodSuffix) } @@ -100,15 +104,14 @@ func (msg *jsonrpcMessage) errorResponse(err error) *jsonrpcMessage { func (msg *jsonrpcMessage) response(result interface{}) *jsonrpcMessage { enc, err := json.Marshal(result) if err != nil { - // TODO: wrap with 'internal server error' - return msg.errorResponse(err) + return msg.errorResponse(&internalServerError{errcodeMarshalError, err.Error()}) } return &jsonrpcMessage{Version: vsn, ID: msg.ID, Result: enc} } func errorMessage(err error) *jsonrpcMessage { msg := &jsonrpcMessage{Version: vsn, ID: null, Error: &jsonError{ - Code: defaultErrorCode, + Code: errcodeDefault, Message: err.Error(), }} ec, ok := err.(Error) diff --git a/rpc/metrics.go b/rpc/metrics.go index 4f166ad1cc073..b1f1284535e26 100644 --- a/rpc/metrics.go +++ b/rpc/metrics.go @@ -18,6 +18,7 @@ package rpc import ( "fmt" + "time" "github.com/ethereum/go-ethereum/metrics" ) @@ -26,14 +27,24 @@ var ( rpcRequestGauge = metrics.NewRegisteredGauge("rpc/requests", nil) successfulRequestGauge = metrics.NewRegisteredGauge("rpc/success", nil) failedRequestGauge = metrics.NewRegisteredGauge("rpc/failure", nil) - rpcServingTimer = metrics.NewRegisteredTimer("rpc/duration/all", nil) + + // serveTimeHistName is the prefix of the per-request serving time histograms. + serveTimeHistName = "rpc/duration" + + rpcServingTimer = metrics.NewRegisteredTimer("rpc/duration/all", nil) ) -func newRPCServingTimer(method string, valid bool) metrics.Timer { - flag := "success" - if !valid { - flag = "failure" +// updateServeTimeHistogram tracks the serving time of a remote RPC call. +func updateServeTimeHistogram(method string, success bool, elapsed time.Duration) { + note := "success" + if !success { + note = "failure" + } + h := fmt.Sprintf("%s/%s/%s", serveTimeHistName, method, note) + sampler := func() metrics.Sample { + return metrics.ResettingSample( + metrics.NewExpDecaySample(1028, 0.015), + ) } - m := fmt.Sprintf("rpc/duration/%s/%s", method, flag) - return metrics.GetOrRegisterTimer(m, nil) + metrics.GetOrRegisterHistogramLazy(h, nil, sampler).Update(elapsed.Microseconds()) } diff --git a/rpc/server.go b/rpc/server.go index babc5688e2648..bf1f71a28e262 100644 --- a/rpc/server.go +++ b/rpc/server.go @@ -160,7 +160,7 @@ type PeerInfo struct { // Address of client. This will usually contain the IP address and port. RemoteAddr string - // Addditional information for HTTP and WebSocket connections. + // Additional information for HTTP and WebSocket connections. HTTP struct { // Protocol version, i.e. "HTTP/1.1". This is not set for WebSocket. Version string diff --git a/rpc/server_test.go b/rpc/server_test.go index d09d31634beeb..c9abe53e52102 100644 --- a/rpc/server_test.go +++ b/rpc/server_test.go @@ -45,7 +45,7 @@ func TestServerRegisterName(t *testing.T) { t.Fatalf("Expected service calc to be registered") } - wantCallbacks := 10 + wantCallbacks := 12 if len(svc.callbacks) != wantCallbacks { t.Errorf("Expected %d callbacks for service 'service', got %d", wantCallbacks, len(svc.callbacks)) } diff --git a/rpc/service.go b/rpc/service.go index bef891ea11252..cfdfba023a0ad 100644 --- a/rpc/service.go +++ b/rpc/service.go @@ -18,7 +18,6 @@ package rpc import ( "context" - "errors" "fmt" "reflect" "runtime" @@ -199,7 +198,7 @@ func (c *callback) call(ctx context.Context, method string, args []reflect.Value buf := make([]byte, size) buf = buf[:runtime.Stack(buf, false)] log.Error("RPC method " + method + " crashed: " + fmt.Sprintf("%v\n%s", err, buf)) - errRes = errors.New("method handler crashed") + errRes = &internalServerError{errcodePanic, "method handler crashed"} } }() // Run the callback. diff --git a/rpc/stdio.go b/rpc/stdio.go index be2bab1c98bd1..ae32db26ef1c7 100644 --- a/rpc/stdio.go +++ b/rpc/stdio.go @@ -32,12 +32,16 @@ func DialStdIO(ctx context.Context) (*Client, error) { // DialIO creates a client which uses the given IO channels func DialIO(ctx context.Context, in io.Reader, out io.Writer) (*Client, error) { - return newClient(ctx, func(_ context.Context) (ServerCodec, error) { + return newClient(ctx, newClientTransportIO(in, out)) +} + +func newClientTransportIO(in io.Reader, out io.Writer) reconnectFunc { + return func(context.Context) (ServerCodec, error) { return NewCodec(stdioConn{ in: in, out: out, }), nil - }) + } } type stdioConn struct { diff --git a/rpc/subscription_test.go b/rpc/subscription_test.go index 54a053dba8052..b2704578291eb 100644 --- a/rpc/subscription_test.go +++ b/rpc/subscription_test.go @@ -48,7 +48,7 @@ func TestNewID(t *testing.T) { func TestSubscriptions(t *testing.T) { var ( - namespaces = []string{"eth", "shh", "bzz"} + namespaces = []string{"eth", "bzz"} service = ¬ificationTestService{} subCount = len(namespaces) notificationCount = 3 @@ -79,7 +79,7 @@ func TestSubscriptions(t *testing.T) { request := map[string]interface{}{ "id": i, "method": fmt.Sprintf("%s_subscribe", namespace), - "version": "2.0", + "jsonrpc": "2.0", "params": []interface{}{"someSubscription", notificationCount, i}, } if err := out.Encode(&request); err != nil { diff --git a/rpc/testdata/internal-error.js b/rpc/testdata/internal-error.js new file mode 100644 index 0000000000000..2ba387401f24e --- /dev/null +++ b/rpc/testdata/internal-error.js @@ -0,0 +1,7 @@ +// These tests trigger various 'internal error' conditions. + +--> {"jsonrpc":"2.0","id":1,"method":"test_marshalError","params": []} +<-- {"jsonrpc":"2.0","id":1,"error":{"code":-32603,"message":"json: error calling MarshalText for type *rpc.MarshalErrObj: marshal error"}} + +--> {"jsonrpc":"2.0","id":2,"method":"test_panic","params": []} +<-- {"jsonrpc":"2.0","id":2,"error":{"code":-32603,"message":"method handler crashed"}} diff --git a/rpc/testdata/invalid-badversion.js b/rpc/testdata/invalid-badversion.js new file mode 100644 index 0000000000000..75b5291dc3f0e --- /dev/null +++ b/rpc/testdata/invalid-badversion.js @@ -0,0 +1,19 @@ +// This test checks processing of messages with invalid Version. + +--> {"jsonrpc":"2.0","id":1,"method":"test_echo","params":["x", 3]} +<-- {"jsonrpc":"2.0","id":1,"result":{"String":"x","Int":3,"Args":null}} + +--> {"jsonrpc":"2.1","id":1,"method":"test_echo","params":["x", 3]} +<-- {"jsonrpc":"2.0","id":1,"error":{"code":-32600,"message":"invalid request"}} + +--> {"jsonrpc":"go-ethereum","id":1,"method":"test_echo","params":["x", 3]} +<-- {"jsonrpc":"2.0","id":1,"error":{"code":-32600,"message":"invalid request"}} + +--> {"jsonrpc":1,"id":1,"method":"test_echo","params":["x", 3]} +<-- {"jsonrpc":"2.0","id":1,"error":{"code":-32600,"message":"invalid request"}} + +--> {"jsonrpc":2.0,"id":1,"method":"test_echo","params":["x", 3]} +<-- {"jsonrpc":"2.0","id":1,"error":{"code":-32600,"message":"invalid request"}} + +--> {"id":1,"method":"test_echo","params":["x", 3]} +<-- {"jsonrpc":"2.0","id":1,"error":{"code":-32600,"message":"invalid request"}} diff --git a/rpc/testservice_test.go b/rpc/testservice_test.go index 253e263289000..8454a40192221 100644 --- a/rpc/testservice_test.go +++ b/rpc/testservice_test.go @@ -70,6 +70,12 @@ func (testError) Error() string { return "testError" } func (testError) ErrorCode() int { return 444 } func (testError) ErrorData() interface{} { return "testError data" } +type MarshalErrObj struct{} + +func (o *MarshalErrObj) MarshalText() ([]byte, error) { + return nil, errors.New("marshal error") +} + func (s *testService) NoArgsRets() {} func (s *testService) Echo(str string, i int, args *echoArgs) echoResult { @@ -114,6 +120,14 @@ func (s *testService) ReturnError() error { return testError{} } +func (s *testService) MarshalError() *MarshalErrObj { + return &MarshalErrObj{} +} + +func (s *testService) Panic() string { + panic("service panic") +} + func (s *testService) CallMeBack(ctx context.Context, method string, args []interface{}) (interface{}, error) { c, ok := ClientFromContext(ctx) if !ok { diff --git a/rpc/types.go b/rpc/types.go index f4d05be48cd47..e3d1a48968211 100644 --- a/rpc/types.go +++ b/rpc/types.go @@ -31,9 +31,9 @@ import ( // API describes the set of methods offered over the RPC interface type API struct { Namespace string // namespace under which the rpc methods of Service are exposed - Version string // api version for DApp's + Version string // deprecated - this field is no longer used, but retained for compatibility Service interface{} // receiver instance which holds the methods - Public bool // indication if the methods must be considered safe for public use + Public bool // deprecated - this field is no longer used, but retained for compatibility Authenticated bool // whether the api should only be available behind authentication. } @@ -61,6 +61,7 @@ type jsonWriter interface { type BlockNumber int64 const ( + SafeBlockNumber = BlockNumber(-4) FinalizedBlockNumber = BlockNumber(-3) PendingBlockNumber = BlockNumber(-2) LatestBlockNumber = BlockNumber(-1) @@ -92,6 +93,9 @@ func (bn *BlockNumber) UnmarshalJSON(data []byte) error { case "finalized": *bn = FinalizedBlockNumber return nil + case "safe": + *bn = SafeBlockNumber + return nil } blckNum, err := hexutil.DecodeUint64(input) @@ -118,6 +122,8 @@ func (bn BlockNumber) MarshalText() ([]byte, error) { return []byte("pending"), nil case FinalizedBlockNumber: return []byte("finalized"), nil + case SafeBlockNumber: + return []byte("safe"), nil default: return hexutil.Uint64(bn).MarshalText() } @@ -168,6 +174,10 @@ func (bnh *BlockNumberOrHash) UnmarshalJSON(data []byte) error { bn := FinalizedBlockNumber bnh.BlockNumber = &bn return nil + case "safe": + bn := SafeBlockNumber + bnh.BlockNumber = &bn + return nil default: if len(input) == 66 { hash := common.Hash{} diff --git a/rpc/websocket.go b/rpc/websocket.go index 28380d8aa4ae0..f2a923446cac3 100644 --- a/rpc/websocket.go +++ b/rpc/websocket.go @@ -181,24 +181,23 @@ func parseOriginURL(origin string) (string, string, string, error) { return scheme, hostname, port, nil } -// DialWebsocketWithDialer creates a new RPC client that communicates with a JSON-RPC server -// that is listening on the given endpoint using the provided dialer. +// DialWebsocketWithDialer creates a new RPC client using WebSocket. +// +// The context is used for the initial connection establishment. It does not +// affect subsequent interactions with the client. +// +// Deprecated: use DialOptions and the WithWebsocketDialer option. func DialWebsocketWithDialer(ctx context.Context, endpoint, origin string, dialer websocket.Dialer) (*Client, error) { - endpoint, header, err := wsClientHeaders(endpoint, origin) + cfg := new(clientConfig) + cfg.wsDialer = &dialer + if origin != "" { + cfg.setHeader("origin", origin) + } + connect, err := newClientTransportWS(endpoint, cfg) if err != nil { return nil, err } - return newClient(ctx, func(ctx context.Context) (ServerCodec, error) { - conn, resp, err := dialer.DialContext(ctx, endpoint, header) - if err != nil { - hErr := wsHandshakeError{err: err} - if resp != nil { - hErr.status = resp.Status - } - return nil, hErr - } - return newWebsocketCodec(conn, endpoint, header), nil - }) + return newClient(ctx, connect) } // DialWebsocket creates a new RPC client that communicates with a JSON-RPC server @@ -207,12 +206,53 @@ func DialWebsocketWithDialer(ctx context.Context, endpoint, origin string, diale // The context is used for the initial connection establishment. It does not // affect subsequent interactions with the client. func DialWebsocket(ctx context.Context, endpoint, origin string) (*Client, error) { - dialer := websocket.Dialer{ - ReadBufferSize: wsReadBuffer, - WriteBufferSize: wsWriteBuffer, - WriteBufferPool: wsBufferPool, + cfg := new(clientConfig) + if origin != "" { + cfg.setHeader("origin", origin) + } + connect, err := newClientTransportWS(endpoint, cfg) + if err != nil { + return nil, err + } + return newClient(ctx, connect) +} + +func newClientTransportWS(endpoint string, cfg *clientConfig) (reconnectFunc, error) { + dialer := cfg.wsDialer + if dialer == nil { + dialer = &websocket.Dialer{ + ReadBufferSize: wsReadBuffer, + WriteBufferSize: wsWriteBuffer, + WriteBufferPool: wsBufferPool, + } + } + + dialURL, header, err := wsClientHeaders(endpoint, "") + if err != nil { + return nil, err + } + for key, values := range cfg.httpHeaders { + header[key] = values + } + + connect := func(ctx context.Context) (ServerCodec, error) { + header := header.Clone() + if cfg.httpAuth != nil { + if err := cfg.httpAuth(header); err != nil { + return nil, err + } + } + conn, resp, err := dialer.DialContext(ctx, dialURL, header) + if err != nil { + hErr := wsHandshakeError{err: err} + if resp != nil { + hErr.status = resp.Status + } + return nil, hErr + } + return newWebsocketCodec(conn, dialURL, header), nil } - return DialWebsocketWithDialer(ctx, endpoint, origin, dialer) + return connect, nil } func wsClientHeaders(endpoint, origin string) (string, http.Header, error) { diff --git a/signer/core/api.go b/signer/core/api.go index f06fbeb76dd13..f10f03d83ac1b 100644 --- a/signer/core/api.go +++ b/signer/core/api.go @@ -319,7 +319,6 @@ func (api *SignerAPI) openTrezor(url accounts.URL) { log.Warn("failed to open wallet", "wallet", url, "err", err) return } - } // startUSBListener starts a listener for USB events, for hardware wallet interaction @@ -612,7 +611,6 @@ func (api *SignerAPI) SignTransaction(ctx context.Context, args apitypes.SendTxA api.UI.OnApprovedTx(response) // ...and to the external caller return &response, nil - } func (api *SignerAPI) SignGnosisSafeTx(ctx context.Context, signerAddress common.MixedcaseAddress, gnosisTx GnosisSafeTx, methodSelector *string) (*GnosisSafeTx, error) { diff --git a/signer/core/api_test.go b/signer/core/api_test.go index ddc2b82eac69b..9bb55bddca317 100644 --- a/signer/core/api_test.go +++ b/signer/core/api_test.go @@ -39,7 +39,7 @@ import ( "github.com/ethereum/go-ethereum/signer/storage" ) -//Used for testing +// Used for testing type headlessUi struct { approveCh chan string // to send approve/deny inputCh chan string // to send password @@ -55,14 +55,13 @@ func (ui *headlessUi) RegisterUIServer(api *core.UIServerAPI) {} func (ui *headlessUi) OnApprovedTx(tx ethapi.SignTransactionResult) {} func (ui *headlessUi) ApproveTx(request *core.SignTxRequest) (core.SignTxResponse, error) { - switch <-ui.approveCh { case "Y": return core.SignTxResponse{request.Transaction, true}, nil case "M": // modify // The headless UI always modifies the transaction old := big.Int(request.Transaction.Value) - newVal := big.NewInt(0).Add(&old, big.NewInt(1)) + newVal := new(big.Int).Add(&old, big.NewInt(1)) request.Transaction.Value = hexutil.Big(*newVal) return core.SignTxResponse{request.Transaction, true}, nil default: @@ -125,7 +124,6 @@ func setup(t *testing.T) (*core.SignerAPI, *headlessUi) { am := core.StartClefAccountManager(tmpDirName(t), true, true, "") api := core.NewSignerAPI(am, 1337, true, ui, db, true, &storage.NoStorage{}) return api, ui - } func createAccount(ui *headlessUi, api *core.SignerAPI, t *testing.T) { ui.approveCh <- "Y" @@ -139,7 +137,6 @@ func createAccount(ui *headlessUi, api *core.SignerAPI, t *testing.T) { } func failCreateAccountWithPassword(ui *headlessUi, api *core.SignerAPI, password string, t *testing.T) { - ui.approveCh <- "Y" // We will be asked three times to provide a suitable password ui.inputCh <- password @@ -169,7 +166,6 @@ func failCreateAccount(ui *headlessUi, api *core.SignerAPI, t *testing.T) { func list(ui *headlessUi, api *core.SignerAPI, t *testing.T) ([]common.Address, error) { ui.approveCh <- "A" return api.List(context.Background()) - } func TestNewAcc(t *testing.T) { @@ -321,5 +317,4 @@ func TestSignTx(t *testing.T) { if bytes.Equal(res.Raw, res2.Raw) { t.Error("Expected tx to be modified by UI") } - } diff --git a/signer/core/apitypes/types.go b/signer/core/apitypes/types.go index f5c2fe2f3db91..2c8907ac822e3 100644 --- a/signer/core/apitypes/types.go +++ b/signer/core/apitypes/types.go @@ -64,7 +64,7 @@ func (vs *ValidationMessages) Info(msg string) { vs.Messages = append(vs.Messages, ValidationInfo{INFO, msg}) } -/// getWarnings returns an error with all messages of type WARN of above, or nil if no warnings were present +// getWarnings returns an error with all messages of type WARN of above, or nil if no warnings were present func (v *ValidationMessages) GetWarnings() error { var messages []string for _, msg := range v.Messages { @@ -251,6 +251,25 @@ type TypedDataDomain struct { Salt string `json:"salt"` } +// TypedDataAndHash is a helper function that calculates a hash for typed data conforming to EIP-712. +// This hash can then be safely used to calculate a signature. +// +// See https://eips.ethereum.org/EIPS/eip-712 for the full specification. +// +// This gives context to the signed typed data and prevents signing of transactions. +func TypedDataAndHash(typedData TypedData) ([]byte, string, error) { + domainSeparator, err := typedData.HashStruct("EIP712Domain", typedData.Domain.Map()) + if err != nil { + return nil, "", err + } + typedDataHash, err := typedData.HashStruct(typedData.PrimaryType, typedData.Message) + if err != nil { + return nil, "", err + } + rawData := fmt.Sprintf("\x19\x01%s%s", string(domainSeparator), string(typedDataHash)) + return crypto.Keccak256([]byte(rawData)), rawData, nil +} + // HashStruct generates a keccak256 hash of the encoding of the provided data func (typedData *TypedData) HashStruct(primaryType string, data TypedDataMessage) (hexutil.Bytes, error) { encodedData, err := typedData.EncodeData(primaryType, data, 1) @@ -526,7 +545,6 @@ func (typedData *TypedData) EncodePrimitiveValue(encType string, encValue interf return math.U256Bytes(b), nil } return nil, fmt.Errorf("unrecognized type '%s'", encType) - } // dataMismatchError generates an error for a mismatch between @@ -653,13 +671,12 @@ func formatPrimitiveValue(encType string, encValue interface{}) (string, error) } if strings.HasPrefix(encType, "bytes") { return fmt.Sprintf("%s", encValue), nil - } if strings.HasPrefix(encType, "uint") || strings.HasPrefix(encType, "int") { if b, err := parseInteger(encType, encValue); err != nil { return "", err } else { - return fmt.Sprintf("%d (0x%x)", b, b), nil + return fmt.Sprintf("%d (%#x)", b, b), nil } } return "", fmt.Errorf("unhandled type %v", encType) @@ -784,6 +801,8 @@ func isPrimitiveTypeValid(primitiveType string) bool { primitiveType == "int32[]" || primitiveType == "int64" || primitiveType == "int64[]" || + primitiveType == "int96" || + primitiveType == "int96[]" || primitiveType == "int128" || primitiveType == "int128[]" || primitiveType == "int256" || @@ -800,6 +819,8 @@ func isPrimitiveTypeValid(primitiveType string) bool { primitiveType == "uint32[]" || primitiveType == "uint64" || primitiveType == "uint64[]" || + primitiveType == "uint96" || + primitiveType == "uint96[]" || primitiveType == "uint128" || primitiveType == "uint128[]" || primitiveType == "uint256" || diff --git a/signer/core/auditlog.go b/signer/core/auditlog.go index 663d6d1317354..a0b292bf714ca 100644 --- a/signer/core/auditlog.go +++ b/signer/core/auditlog.go @@ -110,7 +110,6 @@ func (l *AuditLogger) Version(ctx context.Context) (string, error) { data, err := l.api.Version(ctx) l.log.Info("Version", "type", "response", "data", data, "error", err) return data, err - } func NewAuditLogger(path string, api ExternalAPI) (*AuditLogger, error) { diff --git a/signer/core/cliui.go b/signer/core/cliui.go index 05c60906cc0cd..187eb1390af7a 100644 --- a/signer/core/cliui.go +++ b/signer/core/cliui.go @@ -59,7 +59,6 @@ func (ui *CommandlineUI) readString() string { } func (ui *CommandlineUI) OnInputRequired(info UserInputRequest) (UserInputResponse, error) { - fmt.Printf("## %s\n\n%s\n", info.Title, info.Prompt) defer fmt.Println("-----------------------") if info.IsPassword { @@ -147,7 +146,6 @@ func (ui *CommandlineUI) ApproveTx(request *SignTxRequest) (SignTxResponse, erro fmt.Printf(" * %s : %s\n", m.Typ, m.Message) } fmt.Println() - } fmt.Printf("\n") showMetadata(request.Meta) @@ -209,7 +207,6 @@ func (ui *CommandlineUI) ApproveListing(request *ListRequest) (ListResponse, err // ApproveNewAccount prompt the user for confirmation to create new Account, and reveal to caller func (ui *CommandlineUI) ApproveNewAccount(request *NewAccountRequest) (NewAccountResponse, error) { - ui.mu.Lock() defer ui.mu.Unlock() @@ -245,7 +242,6 @@ func (ui *CommandlineUI) OnApprovedTx(tx ethapi.SignTransactionResult) { } func (ui *CommandlineUI) OnSignerStartup(info StartupInfo) { - fmt.Printf("------- Signer info -------\n") for k, v := range info.Info { fmt.Printf("* %v : %v\n", k, v) diff --git a/signer/core/signed_data.go b/signer/core/signed_data.go index 9bf47be799d8d..c0da22e626623 100644 --- a/signer/core/signed_data.go +++ b/signer/core/signed_data.go @@ -129,7 +129,7 @@ func (api *SignerAPI) determineSignatureFormat(ctx context.Context, contentType { Name: "Full message for signing", Typ: "hexdata", - Value: fmt.Sprintf("0x%x", msg), + Value: fmt.Sprintf("%#x", msg), }, } req = &SignDataRequest{ContentType: mediaType, Rawdata: []byte(msg), Messages: messages, Hash: sighash} @@ -161,7 +161,7 @@ func (api *SignerAPI) determineSignatureFormat(ctx context.Context, contentType { Name: "Clique header", Typ: "clique", - Value: fmt.Sprintf("clique header %d [0x%x]", header.Number, header.Hash()), + Value: fmt.Sprintf("clique header %d [%#x]", header.Number, header.Hash()), }, } // Clique uses V on the form 0 or 1 @@ -169,7 +169,7 @@ func (api *SignerAPI) determineSignatureFormat(ctx context.Context, contentType req = &SignDataRequest{ContentType: mediaType, Rawdata: cliqueRlp, Messages: messages, Hash: sighash} default: // also case TextPlain.Mime: // Calculates an Ethereum ECDSA signature for: - // hash = keccak256("\x19${byteVersion}Ethereum Signed Message:\n${message length}${message}") + // hash = keccak256("\x19Ethereum Signed Message:\n${message length}${message}") // We expect it to be a string if stringData, ok := data.(string); !ok { return nil, useEthereumV, fmt.Errorf("input for text/plain must be an hex-encoded string") @@ -194,7 +194,7 @@ func (api *SignerAPI) determineSignatureFormat(ctx context.Context, contentType return req, useEthereumV, nil } -// SignTextWithValidator signs the given message which can be further recovered +// SignTextValidator signs the given message which can be further recovered // with the given validator. // hash = keccak256("\x19\x00"${address}${data}). func SignTextValidator(validatorData apitypes.ValidatorData) (hexutil.Bytes, string) { @@ -233,23 +233,17 @@ func (api *SignerAPI) SignTypedData(ctx context.Context, addr common.MixedcaseAd // - the signature preimage (hash) func (api *SignerAPI) signTypedData(ctx context.Context, addr common.MixedcaseAddress, typedData apitypes.TypedData, validationMessages *apitypes.ValidationMessages) (hexutil.Bytes, hexutil.Bytes, error) { - domainSeparator, err := typedData.HashStruct("EIP712Domain", typedData.Domain.Map()) + sighash, rawData, err := apitypes.TypedDataAndHash(typedData) if err != nil { return nil, nil, err } - typedDataHash, err := typedData.HashStruct(typedData.PrimaryType, typedData.Message) - if err != nil { - return nil, nil, err - } - rawData := []byte(fmt.Sprintf("\x19\x01%s%s", string(domainSeparator), string(typedDataHash))) - sighash := crypto.Keccak256(rawData) messages, err := typedData.Format() if err != nil { return nil, nil, err } req := &SignDataRequest{ ContentType: apitypes.DataTyped.Mime, - Rawdata: rawData, + Rawdata: []byte(rawData), Messages: messages, Hash: sighash, Address: addr} @@ -271,11 +265,11 @@ func (api *SignerAPI) EcRecover(ctx context.Context, data hexutil.Bytes, sig hex // // Note, this function is compatible with eth_sign and personal_sign. As such it recovers // the address of: - // hash = keccak256("\x19${byteVersion}Ethereum Signed Message:\n${message length}${message}") + // hash = keccak256("\x19Ethereum Signed Message:\n${message length}${message}") // addr = ecrecover(hash, signature) // // Note, the signature must conform to the secp256k1 curve R, S and V values, where - // the V value must be be 27 or 28 for legacy reasons. + // the V value must be 27 or 28 for legacy reasons. // // https://github.com/ethereum/go-ethereum/wiki/Management-APIs#personal_ecRecover if len(sig) != 65 { diff --git a/signer/core/validation_test.go b/signer/core/validation_test.go index 7105691d29c0c..6adaa21afd4ee 100644 --- a/signer/core/validation_test.go +++ b/signer/core/validation_test.go @@ -38,7 +38,6 @@ func TestPasswordValidation(t *testing.T) { if err == nil && test.shouldFail { t.Errorf("password '%v' should fail validation", test.pw) } else if err != nil && !test.shouldFail { - t.Errorf("password '%v' shound not fail validation, but did: %v", test.pw, err) } } diff --git a/signer/fourbyte/validation_test.go b/signer/fourbyte/validation_test.go index c3085696f4326..1b0ab507a8649 100644 --- a/signer/fourbyte/validation_test.go +++ b/signer/fourbyte/validation_test.go @@ -29,11 +29,11 @@ func mixAddr(a string) (*common.MixedcaseAddress, error) { return common.NewMixedcaseAddressFromString(a) } func toHexBig(h string) hexutil.Big { - b := big.NewInt(0).SetBytes(common.FromHex(h)) + b := new(big.Int).SetBytes(common.FromHex(h)) return hexutil.Big(*b) } func toHexUint(h string) hexutil.Uint64 { - b := big.NewInt(0).SetBytes(common.FromHex(h)) + b := new(big.Int).SetBytes(common.FromHex(h)) return hexutil.Uint64(b.Uint64()) } func dummyTxArgs(t txtestcase) *apitypes.SendTxArgs { @@ -53,7 +53,6 @@ func dummyTxArgs(t txtestcase) *apitypes.SendTxArgs { if t.i != "" { a := hexutil.Bytes(common.FromHex(t.i)) input = &a - } return &apitypes.SendTxArgs{ From: *from, diff --git a/signer/rules/rules.go b/signer/rules/rules.go index 6852d86f3ec72..5ed4514e02279 100644 --- a/signer/rules/rules.go +++ b/signer/rules/rules.go @@ -59,6 +59,7 @@ func NewRuleEvaluator(next core.UIClientAPI, jsbackend storage.Storage) (*rulese return c, nil } func (r *rulesetUI) RegisterUIServer(api *core.UIServerAPI) { + r.next.RegisterUIServer(api) // TODO, make it possible to query from js } @@ -67,7 +68,6 @@ func (r *rulesetUI) Init(javascriptRules string) error { return nil } func (r *rulesetUI) execute(jsfunc string, jsarg interface{}) (goja.Value, error) { - // Instantiate a fresh vm engine every time vm := goja.New() diff --git a/signer/rules/rules_test.go b/signer/rules/rules_test.go index 0ab246eeaf7e7..c35da8ecc188c 100644 --- a/signer/rules/rules_test.go +++ b/signer/rules/rules_test.go @@ -44,7 +44,7 @@ Three things can happen: 3. Anything else; other return values [*], method not implemented or exception occurred during processing. This means that the operation will continue to manual processing, via the regular UI method chosen by the user. -[*] Note: Future version of the ruleset may use more complex json-based returnvalues, making it possible to not +[*] Note: Future version of the ruleset may use more complex json-based return values, making it possible to not only respond Approve/Reject/Manual, but also modify responses. For example, choose to list only one, but not all accounts in a list-request. The points above will continue to hold for non-json based responses ("Approve"/"Reject"). @@ -152,7 +152,6 @@ func TestListRequest(t *testing.T) { } func TestSignTxRequest(t *testing.T) { - js := ` function ApproveTx(r){ console.log("transaction.from", r.transaction.from); @@ -243,9 +242,8 @@ func (d *dummyUI) OnApprovedTx(tx ethapi.SignTransactionResult) { func (d *dummyUI) OnSignerStartup(info core.StartupInfo) { } -//TestForwarding tests that the rule-engine correctly dispatches requests to the next caller +// TestForwarding tests that the rule-engine correctly dispatches requests to the next caller func TestForwarding(t *testing.T) { - js := "" ui := &dummyUI{make([]string, 0)} jsBackend := storage.NewEphemeralStorage() @@ -268,11 +266,8 @@ func TestForwarding(t *testing.T) { expCalls := 6 if len(ui.calls) != expCalls { - t.Errorf("Expected %d forwarded calls, got %d: %s", expCalls, len(ui.calls), strings.Join(ui.calls, ",")) - } - } func TestMissingFunc(t *testing.T) { @@ -296,10 +291,8 @@ func TestMissingFunc(t *testing.T) { t.Errorf("Expected missing method to cause non-approval") } t.Logf("Err %v", err) - } func TestStorage(t *testing.T) { - js := ` function testStorage(){ storage.put("mykey", "myvalue") @@ -348,7 +341,6 @@ func TestStorage(t *testing.T) { t.Errorf("Unexpected data, expected '%v', got '%v'", exp, retval) } t.Logf("Err %v", err) - } const ExampleTxWindow = ` @@ -442,14 +434,14 @@ func dummyTx(value hexutil.Big) *core.SignTxRequest { Gas: gas, }, Callinfo: []apitypes.ValidationInfo{ - {Typ: "Warning", Message: "All your base are bellong to us"}, + {Typ: "Warning", Message: "All your base are belong to us"}, }, Meta: core.Metadata{Remote: "remoteip", Local: "localip", Scheme: "inproc"}, } } func dummyTxWithV(value uint64) *core.SignTxRequest { - v := big.NewInt(0).SetUint64(value) + v := new(big.Int).SetUint64(value) h := hexutil.Big(*v) return dummyTx(h) } @@ -469,7 +461,7 @@ func TestLimitWindow(t *testing.T) { return } // 0.3 ether: 429D069189E0000 wei - v := big.NewInt(0).SetBytes(common.Hex2Bytes("0429D069189E0000")) + v := new(big.Int).SetBytes(common.Hex2Bytes("0429D069189E0000")) h := hexutil.Big(*v) // The first three should succeed for i := 0; i < 3; i++ { @@ -544,11 +536,10 @@ func (d *dontCallMe) OnApprovedTx(tx ethapi.SignTransactionResult) { d.t.Fatalf("Did not expect next-handler to be called") } -//TestContextIsCleared tests that the rule-engine does not retain variables over several requests. +// TestContextIsCleared tests that the rule-engine does not retain variables over several requests. // if it does, that would be bad since developers may rely on that to store data, // instead of using the disk-based data storage func TestContextIsCleared(t *testing.T) { - js := ` function ApproveTx(){ if (typeof foobar == 'undefined') { @@ -580,7 +571,6 @@ func TestContextIsCleared(t *testing.T) { } func TestSignData(t *testing.T) { - js := `function ApproveListing(){ return "Approve" } diff --git a/signer/storage/aes_gcm_storage.go b/signer/storage/aes_gcm_storage.go index f09bfa7d4f06a..928d643dd618b 100644 --- a/signer/storage/aes_gcm_storage.go +++ b/signer/storage/aes_gcm_storage.go @@ -143,7 +143,7 @@ func (s *AESEncryptedStorage) writeEncryptedStorage(creds map[string]storedCrede // encrypt encrypts plaintext with the given key, with additional data // The 'additionalData' is used to place the (plaintext) KV-store key into the V, -// to prevent the possibility to alter a K, or swap two entries in the KV store with eachother. +// to prevent the possibility to alter a K, or swap two entries in the KV store with each other. func encrypt(key []byte, plaintext []byte, additionalData []byte) ([]byte, []byte, error) { block, err := aes.NewCipher(key) if err != nil { diff --git a/signer/storage/aes_gcm_storage_test.go b/signer/storage/aes_gcm_storage_test.go index a2a95d9deedf6..e1fea59280a82 100644 --- a/signer/storage/aes_gcm_storage_test.go +++ b/signer/storage/aes_gcm_storage_test.go @@ -51,7 +51,6 @@ func TestEncryption(t *testing.T) { } func TestFileStorage(t *testing.T) { - a := map[string]storedCredential{ "secret": { Iv: common.Hex2Bytes("cdb30036279601aeee60f16b"), diff --git a/tests/block_test_util.go b/tests/block_test_util.go index 76f0b880b4a84..313a703fae8f3 100644 --- a/tests/block_test_util.go +++ b/tests/block_test_util.go @@ -106,7 +106,8 @@ func (t *BlockTest) Run(snapshotter bool) error { // import pre accounts & construct test genesis block & state root db := rawdb.NewMemoryDatabase() - gblock, err := t.genesis(config).Commit(db) + gspec := t.genesis(config) + gblock, err := gspec.Commit(db) if err != nil { return err } @@ -127,7 +128,7 @@ func (t *BlockTest) Run(snapshotter bool) error { cache.SnapshotLimit = 1 cache.SnapshotWait = true } - chain, err := core.NewBlockChain(db, cache, config, engine, vm.Config{}, nil, nil) + chain, err := core.NewBlockChain(db, cache, gspec, nil, engine, vm.Config{}, nil, nil) if err != nil { return err } @@ -174,17 +175,18 @@ func (t *BlockTest) genesis(config *params.ChainConfig) *core.Genesis { } } -/* See https://github.com/ethereum/tests/wiki/Blockchain-Tests-II +/* +See https://github.com/ethereum/tests/wiki/Blockchain-Tests-II - Whether a block is valid or not is a bit subtle, it's defined by presence of - blockHeader, transactions and uncleHeaders fields. If they are missing, the block is - invalid and we must verify that we do not accept it. + Whether a block is valid or not is a bit subtle, it's defined by presence of + blockHeader, transactions and uncleHeaders fields. If they are missing, the block is + invalid and we must verify that we do not accept it. - Since some tests mix valid and invalid blocks we need to check this for every block. + Since some tests mix valid and invalid blocks we need to check this for every block. - If a block is invalid it does not necessarily fail the test, if it's invalidness is - expected we are expected to ignore it and continue processing and then validate the - post state. + If a block is invalid it does not necessarily fail the test, if it's invalidness is + expected we are expected to ignore it and continue processing and then validate the + post state. */ func (t *BlockTest) insertBlocks(blockchain *core.BlockChain) ([]btBlock, error) { validBlocks := make([]btBlock, 0) diff --git a/tests/difficulty_test.go b/tests/difficulty_test.go index 192dff12cc97c..af9b0f86bd9a5 100644 --- a/tests/difficulty_test.go +++ b/tests/difficulty_test.go @@ -52,11 +52,8 @@ func TestDifficulty(t *testing.T) { // files are 2 years old, contains strange values dt.skipLoad("difficultyCustomHomestead\\.json") - dt.skipLoad("difficultyMorden\\.json") - dt.skipLoad("difficultyOlimpic\\.json") dt.config("Ropsten", *params.RopstenChainConfig) - dt.config("Morden", *params.RopstenChainConfig) dt.config("Frontier", params.ChainConfig{}) dt.config("Homestead", params.ChainConfig{ @@ -79,6 +76,9 @@ func TestDifficulty(t *testing.T) { dt.config("EIP4345", params.ChainConfig{ ArrowGlacierBlock: big.NewInt(0), }) + dt.config("EIP5133", params.ChainConfig{ + GrayGlacierBlock: big.NewInt(0), + }) dt.config("difficulty.json", mainnetChainConfig) dt.walk(t, difficultyTestDir, func(t *testing.T, name string, test *DifficultyTest) { diff --git a/tests/difficulty_test_util.go b/tests/difficulty_test_util.go index bda5a9611be89..62b978f9ef2b0 100644 --- a/tests/difficulty_test_util.go +++ b/tests/difficulty_test_util.go @@ -65,5 +65,4 @@ func (test *DifficultyTest) Run(config *params.ChainConfig) error { test.CurrentTimestamp, test.CurrentBlockNumber, actual, exp) } return nil - } diff --git a/tests/fuzzers/bls12381/bls12381_fuzz.go b/tests/fuzzers/bls12381/bls12381_fuzz.go index c511c65011328..a5b4b9e798569 100644 --- a/tests/fuzzers/bls12381/bls12381_fuzz.go +++ b/tests/fuzzers/bls12381/bls12381_fuzz.go @@ -29,6 +29,7 @@ import ( gnark "github.com/consensys/gnark-crypto/ecc/bls12-381" "github.com/consensys/gnark-crypto/ecc/bls12-381/fp" "github.com/consensys/gnark-crypto/ecc/bls12-381/fr" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto/bls12381" blst "github.com/supranational/blst/bindings/go" ) @@ -64,18 +65,41 @@ func FuzzCrossPairing(data []byte) int { panic("pairing mismatch gnark / geth ") } - var b []byte - ctx := blst.PairingCtx(false, b) // compute pairing using blst - blst.PairingRawAggregate(ctx, blG2, blG1) - blstResult := blst.PairingAsFp12(ctx) - if !(bytes.Equal(blstResult.ToBendian(), bls12381.NewGT().ToBytes(kResult))) { - panic("pairing mismatch blst / geth ") + blstResult := blst.Fp12MillerLoop(blG2, blG1) + blstResult.FinalExp() + res := massageBLST(blstResult.ToBendian()) + if !(bytes.Equal(res, bls12381.NewGT().ToBytes(kResult))) { + panic("pairing mismatch blst / geth") } return 1 } +func massageBLST(in []byte) []byte { + out := make([]byte, len(in)) + len := 12 * 48 + // 1 + copy(out[0:], in[len-1*48:len]) + copy(out[1*48:], in[len-2*48:len-1*48]) + // 2 + copy(out[6*48:], in[len-3*48:len-2*48]) + copy(out[7*48:], in[len-4*48:len-3*48]) + // 3 + copy(out[2*48:], in[len-5*48:len-4*48]) + copy(out[3*48:], in[len-6*48:len-5*48]) + // 4 + copy(out[8*48:], in[len-7*48:len-6*48]) + copy(out[9*48:], in[len-8*48:len-7*48]) + // 5 + copy(out[4*48:], in[len-9*48:len-8*48]) + copy(out[5*48:], in[len-10*48:len-9*48]) + // 6 + copy(out[10*48:], in[len-11*48:len-10*48]) + copy(out[11*48:], in[len-12*48:len-11*48]) + return out +} + func FuzzCrossG1Add(data []byte) int { input := bytes.NewReader(data) @@ -227,10 +251,8 @@ func getG1Points(input io.Reader) (*bls12381.PointG1, *gnark.G1Affine, *blst.P1A } // marshal gnark point -> blst point - var p1 *blst.P1Affine - var scalar *blst.Scalar - scalar.Deserialize(s.Bytes()) - p1.From(scalar) + scalar := new(blst.Scalar).FromBEndian(common.LeftPadBytes(s.Bytes(), 32)) + p1 := new(blst.P1Affine).From(scalar) if !bytes.Equal(p1.Serialize(), cpBytes) { panic("bytes(blst.G1) != bytes(geth.G1)") } @@ -262,10 +284,9 @@ func getG2Points(input io.Reader) (*bls12381.PointG2, *gnark.G2Affine, *blst.P2A } // marshal gnark point -> blst point - var p2 *blst.P2Affine - var scalar *blst.Scalar - scalar.Deserialize(s.Bytes()) - p2.From(scalar) + // Left pad the scalar to 32 bytes + scalar := new(blst.Scalar).FromBEndian(common.LeftPadBytes(s.Bytes(), 32)) + p2 := new(blst.P2Affine).From(scalar) if !bytes.Equal(p2.Serialize(), cpBytes) { panic("bytes(blst.G2) != bytes(geth.G2)") } diff --git a/tests/fuzzers/bls12381/precompile_fuzzer.go b/tests/fuzzers/bls12381/precompile_fuzzer.go index bc3c456526039..cab2bcba38632 100644 --- a/tests/fuzzers/bls12381/precompile_fuzzer.go +++ b/tests/fuzzers/bls12381/precompile_fuzzer.go @@ -70,12 +70,14 @@ func checkInput(id byte, inputLen int) bool { panic("programmer error") } -// The fuzzer functions must return -// 1 if the fuzzer should increase priority of the -// given input during subsequent fuzzing (for example, the input is lexically -// correct and was parsed successfully); -// -1 if the input must not be added to corpus even if gives new coverage; and -// 0 otherwise +// The function must return +// +// - 1 if the fuzzer should increase priority of the +// given input during subsequent fuzzing (for example, the input is lexically +// correct and was parsed successfully); +// - -1 if the input must not be added to corpus even if gives new coverage; and +// - 0 otherwise +// // other values are reserved for future use. func fuzz(id byte, data []byte) int { // Even on bad input, it should not crash, so we still test the gas calc diff --git a/tests/fuzzers/difficulty/difficulty-fuzz.go b/tests/fuzzers/difficulty/difficulty-fuzz.go index 58936fcd80b10..5612a4e706605 100644 --- a/tests/fuzzers/difficulty/difficulty-fuzz.go +++ b/tests/fuzzers/difficulty/difficulty-fuzz.go @@ -30,7 +30,6 @@ import ( type fuzzer struct { input io.Reader exhausted bool - debugging bool } func (f *fuzzer) read(size int) []byte { @@ -68,11 +67,13 @@ func (f *fuzzer) readBool() bool { } // The function must return -// 1 if the fuzzer should increase priority of the -// given input during subsequent fuzzing (for example, the input is lexically -// correct and was parsed successfully); -// -1 if the input must not be added to corpus even if gives new coverage; and -// 0 otherwise +// +// - 1 if the fuzzer should increase priority of the +// given input during subsequent fuzzing (for example, the input is lexically +// correct and was parsed successfully); +// - -1 if the input must not be added to corpus even if gives new coverage; and +// - 0 otherwise +// // other values are reserved for future use. func Fuzz(data []byte) int { f := fuzzer{ diff --git a/tests/fuzzers/les/les-fuzzer.go b/tests/fuzzers/les/les-fuzzer.go index 3e10171873452..d3a25a31c0847 100644 --- a/tests/fuzzers/les/les-fuzzer.go +++ b/tests/fuzzers/les/les-fuzzer.go @@ -54,15 +54,13 @@ var ( ) func makechain() (bc *core.BlockChain, addrHashes, txHashes []common.Hash) { - db := rawdb.NewMemoryDatabase() - gspec := core.Genesis{ + gspec := &core.Genesis{ Config: params.TestChainConfig, Alloc: core.GenesisAlloc{bankAddr: {Balance: bankFunds}}, GasLimit: 100000000, } - genesis := gspec.MustCommit(db) signer := types.HomesteadSigner{} - blocks, _ := core.GenerateChain(gspec.Config, genesis, ethash.NewFaker(), db, testChainLen, + _, blocks, _ := core.GenerateChainWithGenesis(gspec, ethash.NewFaker(), testChainLen, func(i int, gen *core.BlockGen) { var ( tx *types.Transaction @@ -80,7 +78,7 @@ func makechain() (bc *core.BlockChain, addrHashes, txHashes []common.Hash) { addrHashes = append(addrHashes, crypto.Keccak256Hash(addr[:])) txHashes = append(txHashes, tx.Hash()) }) - bc, _ = core.NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) + bc, _ = core.NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) if _, err := bc.InsertChain(blocks); err != nil { panic(err) } @@ -88,8 +86,8 @@ func makechain() (bc *core.BlockChain, addrHashes, txHashes []common.Hash) { } func makeTries() (chtTrie *trie.Trie, bloomTrie *trie.Trie, chtKeys, bloomKeys [][]byte) { - chtTrie, _ = trie.New(common.Hash{}, trie.NewDatabase(rawdb.NewMemoryDatabase())) - bloomTrie, _ = trie.New(common.Hash{}, trie.NewDatabase(rawdb.NewMemoryDatabase())) + chtTrie = trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase())) + bloomTrie = trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase())) for i := 0; i < testChainLen; i++ { // The element in CHT is -> key := make([]byte, 8) diff --git a/tests/fuzzers/rangeproof/rangeproof-fuzzer.go b/tests/fuzzers/rangeproof/rangeproof-fuzzer.go index 18717e70d0012..16242a66ec6ee 100644 --- a/tests/fuzzers/rangeproof/rangeproof-fuzzer.go +++ b/tests/fuzzers/rangeproof/rangeproof-fuzzer.go @@ -62,7 +62,7 @@ func (f *fuzzer) readInt() uint64 { } func (f *fuzzer) randomTrie(n int) (*trie.Trie, map[string]*kv) { - trie, _ := trie.New(common.Hash{}, trie.NewDatabase(rawdb.NewMemoryDatabase())) + trie := trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase())) vals := make(map[string]*kv) size := f.readInt() // Fill it with some fluff @@ -163,7 +163,6 @@ func (f *fuzzer) fuzz() int { // Modify something in the proof db // add stuff to proof db // drop stuff from proof db - } if f.exhausted { break @@ -181,11 +180,14 @@ func (f *fuzzer) fuzz() int { } // The function must return -// 1 if the fuzzer should increase priority of the -// given input during subsequent fuzzing (for example, the input is lexically -// correct and was parsed successfully); -// -1 if the input must not be added to corpus even if gives new coverage; and -// 0 otherwise; other values are reserved for future use. +// +// - 1 if the fuzzer should increase priority of the +// given input during subsequent fuzzing (for example, the input is lexically +// correct and was parsed successfully); +// - -1 if the input must not be added to corpus even if gives new coverage; and +// - 0 otherwise +// +// other values are reserved for future use. func Fuzz(input []byte) int { if len(input) < 100 { return 0 diff --git a/tests/fuzzers/rlp/rlp_fuzzer.go b/tests/fuzzers/rlp/rlp_fuzzer.go index 18b36287b53c2..ac02e1651d446 100644 --- a/tests/fuzzers/rlp/rlp_fuzzer.go +++ b/tests/fuzzers/rlp/rlp_fuzzer.go @@ -40,6 +40,9 @@ func Fuzz(input []byte) int { if len(input) == 0 { return 0 } + if len(input) > 500*1024 { + return 0 + } var i int { diff --git a/tests/fuzzers/snap/fuzz_handler.go b/tests/fuzzers/snap/fuzz_handler.go index 1ae61df29dc71..2e5dcd6e29a07 100644 --- a/tests/fuzzers/snap/fuzz_handler.go +++ b/tests/fuzzers/snap/fuzz_handler.go @@ -39,7 +39,6 @@ import ( var trieRoot common.Hash func getChain() *core.BlockChain { - db := rawdb.NewMemoryDatabase() ga := make(core.GenesisAlloc, 1000) var a = make([]byte, 20) var mkStorage = func(k, v int) (common.Hash, common.Hash) { @@ -62,13 +61,11 @@ func getChain() *core.BlockChain { } ga[common.BytesToAddress(a)] = acc } - gspec := core.Genesis{ + gspec := &core.Genesis{ Config: params.TestChainConfig, Alloc: ga, } - genesis := gspec.MustCommit(db) - blocks, _ := core.GenerateChain(gspec.Config, genesis, ethash.NewFaker(), db, 2, - func(i int, gen *core.BlockGen) {}) + _, blocks, _ := core.GenerateChainWithGenesis(gspec, ethash.NewFaker(), 2, func(i int, gen *core.BlockGen) {}) cacheConf := &core.CacheConfig{ TrieCleanLimit: 0, TrieDirtyLimit: 0, @@ -79,7 +76,7 @@ func getChain() *core.BlockChain { SnapshotWait: true, } trieRoot = blocks[len(blocks)-1].Root() - bc, _ := core.NewBlockChain(db, cacheConf, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) + bc, _ := core.NewBlockChain(rawdb.NewMemoryDatabase(), cacheConf, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) if _, err := bc.InsertChain(blocks); err != nil { panic(err) } diff --git a/tests/fuzzers/stacktrie/trie_fuzzer.go b/tests/fuzzers/stacktrie/trie_fuzzer.go index 9ed8bcbc51d58..95a1fc464e075 100644 --- a/tests/fuzzers/stacktrie/trie_fuzzer.go +++ b/tests/fuzzers/stacktrie/trie_fuzzer.go @@ -25,7 +25,6 @@ import ( "io" "sort" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/trie" "golang.org/x/crypto/sha3" @@ -115,11 +114,13 @@ func (k kvs) Swap(i, j int) { } // The function must return -// 1 if the fuzzer should increase priority of the -// given input during subsequent fuzzing (for example, the input is lexically -// correct and was parsed successfully); -// -1 if the input must not be added to corpus even if gives new coverage; and -// 0 otherwise +// +// - 1 if the fuzzer should increase priority of the +// given input during subsequent fuzzing (for example, the input is lexically +// correct and was parsed successfully); +// - -1 if the input must not be added to corpus even if gives new coverage; and +// - 0 otherwise +// // other values are reserved for future use. func Fuzz(data []byte) int { f := fuzzer{ @@ -139,12 +140,11 @@ func Debug(data []byte) int { } func (f *fuzzer) fuzz() int { - // This spongeDb is used to check the sequence of disk-db-writes var ( spongeA = &spongeDb{sponge: sha3.NewLegacyKeccak256()} dbA = trie.NewDatabase(spongeA) - trieA, _ = trie.New(common.Hash{}, dbA) + trieA = trie.NewEmpty(dbA) spongeB = &spongeDb{sponge: sha3.NewLegacyKeccak256()} trieB = trie.NewStackTrie(spongeB) vals kvs @@ -175,10 +175,13 @@ func (f *fuzzer) fuzz() int { return 0 } // Flush trie -> database - rootA, _, err := trieA.Commit(nil) + rootA, nodes, err := trieA.Commit(false) if err != nil { panic(err) } + if nodes != nil { + dbA.Update(trie.NewWithNodeSet(nodes)) + } // Flush memdb -> disk (sponge) dbA.Commit(rootA, false, nil) @@ -186,7 +189,7 @@ func (f *fuzzer) fuzz() int { sort.Sort(vals) for _, kv := range vals { if f.debugging { - fmt.Printf("{\"0x%x\" , \"0x%x\"} // stacktrie.Update\n", kv.k, kv.v) + fmt.Printf("{\"%#x\" , \"%#x\"} // stacktrie.Update\n", kv.k, kv.v) } trieB.Update(kv.k, kv.v) } diff --git a/tests/fuzzers/trie/trie-fuzzer.go b/tests/fuzzers/trie/trie-fuzzer.go index e993af47cf20c..8467bdafa6b62 100644 --- a/tests/fuzzers/trie/trie-fuzzer.go +++ b/tests/fuzzers/trie/trie-fuzzer.go @@ -21,7 +21,6 @@ import ( "encoding/binary" "fmt" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/ethdb/memorydb" "github.com/ethereum/go-ethereum/trie" ) @@ -51,9 +50,8 @@ const ( opUpdate = iota opDelete opGet - opCommit opHash - opReset + opCommit opItercheckhash opProve opMax // boundary value, not an actual op @@ -84,11 +82,9 @@ func (ds *dataSource) Ended() bool { } func Generate(input []byte) randTest { - var allKeys [][]byte r := newDataSource(input) genKey := func() []byte { - if len(allKeys) < 2 || r.readByte() < 0x0f { // new key key := make([]byte, r.readByte()%50) @@ -103,7 +99,6 @@ func Generate(input []byte) randTest { var steps randTest for i := 0; !r.Ended(); i++ { - step := randTestStep{op: int(r.readByte()) % opMax} switch step.op { case opUpdate: @@ -123,11 +118,13 @@ func Generate(input []byte) randTest { } // The function must return -// 1 if the fuzzer should increase priority of the -// given input during subsequent fuzzing (for example, the input is lexically -// correct and was parsed successfully); -// -1 if the input must not be added to corpus even if gives new coverage; and -// 0 otherwise +// +// - 1 if the fuzzer should increase priority of the +// given input during subsequent fuzzing (for example, the input is lexically +// correct and was parsed successfully); +// - -1 if the input must not be added to corpus even if gives new coverage; and +// - 0 otherwise +// // other values are reserved for future use. func Fuzz(input []byte) int { program := Generate(input) @@ -141,10 +138,9 @@ func Fuzz(input []byte) int { } func runRandTest(rt randTest) error { - triedb := trie.NewDatabase(memorydb.New()) - tr, _ := trie.New(common.Hash{}, triedb) + tr := trie.NewEmpty(triedb) values := make(map[string]string) // tracks content of the trie for i, step := range rt { @@ -159,24 +155,27 @@ func runRandTest(rt randTest) error { v := tr.Get(step.key) want := values[string(step.key)] if string(v) != want { - rt[i].err = fmt.Errorf("mismatch for key 0x%x, got 0x%x want 0x%x", step.key, v, want) + rt[i].err = fmt.Errorf("mismatch for key %#x, got %#x want %#x", step.key, v, want) } - case opCommit: - _, _, rt[i].err = tr.Commit(nil) case opHash: tr.Hash() - case opReset: - hash, _, err := tr.Commit(nil) + case opCommit: + hash, nodes, err := tr.Commit(false) if err != nil { return err } - newtr, err := trie.New(hash, triedb) + if nodes != nil { + if err := triedb.Update(trie.NewWithNodeSet(nodes)); err != nil { + return err + } + } + newtr, err := trie.New(trie.TrieID(hash), triedb) if err != nil { return err } tr = newtr case opItercheckhash: - checktr, _ := trie.New(common.Hash{}, triedb) + checktr := trie.NewEmpty(triedb) it := trie.NewIterator(tr.NodeIterator(nil)) for it.Next() { checktr.Update(it.Key, it.Value) diff --git a/tests/init.go b/tests/init.go index 52277e8416423..87ffc65a677c9 100644 --- a/tests/init.go +++ b/tests/init.go @@ -197,6 +197,22 @@ var Forks = map[string]*params.ChainConfig{ LondonBlock: big.NewInt(0), ArrowGlacierBlock: big.NewInt(0), }, + "GrayGlacier": { + ChainID: big.NewInt(1), + HomesteadBlock: big.NewInt(0), + EIP150Block: big.NewInt(0), + EIP155Block: big.NewInt(0), + EIP158Block: big.NewInt(0), + ByzantiumBlock: big.NewInt(0), + ConstantinopleBlock: big.NewInt(0), + PetersburgBlock: big.NewInt(0), + IstanbulBlock: big.NewInt(0), + MuirGlacierBlock: big.NewInt(0), + BerlinBlock: big.NewInt(0), + LondonBlock: big.NewInt(0), + ArrowGlacierBlock: big.NewInt(0), + GrayGlacierBlock: big.NewInt(0), + }, "Merged": { ChainID: big.NewInt(1), HomesteadBlock: big.NewInt(0), @@ -211,7 +227,7 @@ var Forks = map[string]*params.ChainConfig{ BerlinBlock: big.NewInt(0), LondonBlock: big.NewInt(0), ArrowGlacierBlock: big.NewInt(0), - MergeForkBlock: big.NewInt(0), + MergeNetsplitBlock: big.NewInt(0), TerminalTotalDifficulty: big.NewInt(0), }, } diff --git a/tests/init_test.go b/tests/init_test.go index 218634966d83f..9d315f9511c97 100644 --- a/tests/init_test.go +++ b/tests/init_test.go @@ -116,6 +116,8 @@ func (tm *testMatcher) skipLoad(pattern string) { } // fails adds an expected failure for tests matching the pattern. +// +//nolint:unused func (tm *testMatcher) fails(pattern string, reason string) { if reason == "" { panic("empty fail reason") diff --git a/tests/rlp_test_util.go b/tests/rlp_test_util.go index 9069ec55a15dc..15acb3a244f77 100644 --- a/tests/rlp_test_util.go +++ b/tests/rlp_test_util.go @@ -124,7 +124,7 @@ func translateJSON(v interface{}) interface{} { func checkDecodeFromJSON(s *rlp.Stream, exp interface{}) error { switch exp := exp.(type) { case uint64: - i, err := s.Uint() + i, err := s.Uint64() if err != nil { return addStack("Uint", exp, err) } diff --git a/tests/state_test.go b/tests/state_test.go index d2c92b211cd17..5c605f6722b1f 100644 --- a/tests/state_test.go +++ b/tests/state_test.go @@ -26,6 +26,7 @@ import ( "reflect" "strings" "testing" + "time" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" @@ -57,12 +58,12 @@ func TestState(t *testing.T) { // Broken tests: // Expected failures: - //st.fails(`^stRevertTest/RevertPrecompiledTouch(_storage)?\.json/Byzantium/0`, "bug in test") - //st.fails(`^stRevertTest/RevertPrecompiledTouch(_storage)?\.json/Byzantium/3`, "bug in test") - //st.fails(`^stRevertTest/RevertPrecompiledTouch(_storage)?\.json/Constantinople/0`, "bug in test") - //st.fails(`^stRevertTest/RevertPrecompiledTouch(_storage)?\.json/Constantinople/3`, "bug in test") - //st.fails(`^stRevertTest/RevertPrecompiledTouch(_storage)?\.json/ConstantinopleFix/0`, "bug in test") - //st.fails(`^stRevertTest/RevertPrecompiledTouch(_storage)?\.json/ConstantinopleFix/3`, "bug in test") + // st.fails(`^stRevertTest/RevertPrecompiledTouch(_storage)?\.json/Byzantium/0`, "bug in test") + // st.fails(`^stRevertTest/RevertPrecompiledTouch(_storage)?\.json/Byzantium/3`, "bug in test") + // st.fails(`^stRevertTest/RevertPrecompiledTouch(_storage)?\.json/Constantinople/0`, "bug in test") + // st.fails(`^stRevertTest/RevertPrecompiledTouch(_storage)?\.json/Constantinople/3`, "bug in test") + // st.fails(`^stRevertTest/RevertPrecompiledTouch(_storage)?\.json/ConstantinopleFix/0`, "bug in test") + // st.fails(`^stRevertTest/RevertPrecompiledTouch(_storage)?\.json/ConstantinopleFix/3`, "bug in test") // For Istanbul, older tests were moved into LegacyTests for _, dir := range []string{ @@ -78,10 +79,6 @@ func TestState(t *testing.T) { t.Run(key+"/trie", func(t *testing.T) { withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error { _, _, err := test.Run(subtest, vmconfig, false) - if err != nil && len(test.json.Post[subtest.Fork][subtest.Index].ExpectException) > 0 { - // Ignore expected errors (TODO MariusVanDerWijden check error string) - return nil - } return st.checkFailure(t, err) }) }) @@ -93,10 +90,6 @@ func TestState(t *testing.T) { return err } } - if err != nil && len(test.json.Post[subtest.Fork][subtest.Index].ExpectException) > 0 { - // Ignore expected errors (TODO MariusVanDerWijden check error string) - return nil - } return st.checkFailure(t, err) }) }) @@ -174,6 +167,7 @@ func runBenchmarkFile(b *testing.B, path string) { return } for _, t := range m { + t := t runBenchmark(b, &t) } } @@ -191,12 +185,14 @@ func runBenchmark(b *testing.B, t *StateTest) { b.Error(err) return } + var rules = config.Rules(new(big.Int), false) + vmconfig.ExtraEips = eips - block := t.genesis(config).ToBlock(nil) + block := t.genesis(config).ToBlock() _, statedb := MakePreState(rawdb.NewMemoryDatabase(), t.json.Pre, false) var baseFee *big.Int - if config.IsLondon(new(big.Int)) { + if rules.IsLondon { baseFee = t.json.Env.BaseFee if baseFee == nil { // Retesteth uses `0x10` for genesis baseFee. Therefore, it defaults to @@ -237,18 +233,40 @@ func runBenchmark(b *testing.B, t *StateTest) { sender := vm.NewContract(vm.AccountRef(msg.From()), vm.AccountRef(msg.From()), nil, 0) + var ( + gasUsed uint64 + elapsed uint64 + refund uint64 + ) b.ResetTimer() for n := 0; n < b.N; n++ { - // Execute the message. snapshot := statedb.Snapshot() - _, _, err = evm.Call(sender, *msg.To(), msg.Data(), msg.Gas(), msg.Value()) + if rules.IsBerlin { + statedb.PrepareAccessList(msg.From(), msg.To(), vm.ActivePrecompiles(rules), msg.AccessList()) + } + b.StartTimer() + start := time.Now() + + // Execute the message. + _, leftOverGas, err := evm.Call(sender, *msg.To(), msg.Data(), msg.Gas(), msg.Value()) if err != nil { b.Error(err) return } + + b.StopTimer() + elapsed += uint64(time.Since(start)) + refund += statedb.GetRefund() + gasUsed += msg.Gas() - leftOverGas + statedb.RevertToSnapshot(snapshot) } - + if elapsed < 1 { + elapsed = 1 + } + // Keep it as uint64, multiply 100 to get two digit float later + mgasps := (100 * 1000 * (gasUsed - refund)) / elapsed + b.ReportMetric(float64(mgasps)/100, "mgas/s") }) } } diff --git a/tests/state_test_util.go b/tests/state_test_util.go index f6d8e15001d87..838e85dca2b70 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -159,11 +159,39 @@ func (t *StateTest) Subtests() []StateSubtest { return sub } +// checkError checks if the error returned by the state transition matches any expected error. +// A failing expectation returns a wrapped version of the original error, if any, +// or a new error detailing the failing expectation. +// This function does not return or modify the original error, it only evaluates and returns expectations for the error. +func (t *StateTest) checkError(subtest StateSubtest, err error) error { + expectedError := t.json.Post[subtest.Fork][subtest.Index].ExpectException + if err == nil && expectedError == "" { + return nil + } + if err == nil && expectedError != "" { + return fmt.Errorf("expected error %q, got no error", expectedError) + } + if err != nil && expectedError == "" { + return fmt.Errorf("unexpected error: %w", err) + } + if err != nil && expectedError != "" { + // Ignore expected errors (TODO MariusVanDerWijden check error string) + return nil + } + return nil +} + // Run executes a specific subtest and verifies the post-state and logs func (t *StateTest) Run(subtest StateSubtest, vmconfig vm.Config, snapshotter bool) (*snapshot.Tree, *state.StateDB, error) { snaps, statedb, root, err := t.RunNoVerify(subtest, vmconfig, snapshotter) + if checkedErr := t.checkError(subtest, err); checkedErr != nil { + return snaps, statedb, checkedErr + } + // The error has been checked; if it was unexpected, it's already returned. if err != nil { - return snaps, statedb, err + // Here, an error exists but it was expected. + // We do not check the post state or logs. + return snaps, statedb, nil } post := t.json.Post[subtest.Fork][subtest.Index] // N.B: We need to do this in a two-step process, because the first Commit takes care @@ -184,7 +212,7 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh return nil, nil, common.Hash{}, UnsupportedForkError{subtest.Fork} } vmconfig.ExtraEips = eips - block := t.genesis(config).ToBlock(nil) + block := t.genesis(config).ToBlock() snaps, statedb := MakePreState(rawdb.NewMemoryDatabase(), t.json.Pre, snapshotter) var baseFee *big.Int @@ -220,7 +248,8 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh context := core.NewEVMBlockContext(block.Header(), nil, &t.json.Env.Coinbase) context.GetHash = vmTestBlockHash context.BaseFee = baseFee - if t.json.Env.Random != nil { + context.Random = nil + if config.IsLondon(new(big.Int)) && t.json.Env.Random != nil { rnd := common.BigToHash(t.json.Env.Random) context.Random = &rnd context.Difficulty = big.NewInt(0) @@ -230,7 +259,8 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh snapshot := statedb.Snapshot() gaspool := new(core.GasPool) gaspool.AddGas(block.GasLimit()) - if _, err := core.ApplyMessage(evm, msg, gaspool); err != nil { + _, err = core.ApplyMessage(evm, msg, gaspool) + if err != nil { statedb.RevertToSnapshot(snapshot) } // Add 0-value mining reward. This only makes a difference in the cases @@ -243,7 +273,7 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh statedb.Commit(config.IsEIP158(block.Number())) // And _now_ get the state root root := statedb.IntermediateRoot(config.IsEIP158(block.Number())) - return snaps, statedb, root, nil + return snaps, statedb, root, err } func (t *StateTest) gasLimit(subtest StateSubtest) uint64 { @@ -266,7 +296,13 @@ func MakePreState(db ethdb.Database, accounts core.GenesisAlloc, snapshotter boo var snaps *snapshot.Tree if snapshotter { - snaps, _ = snapshot.New(db, sdb.TrieDB(), 1, root, false, true, false) + snapconfig := snapshot.Config{ + CacheSize: 1, + Recovery: false, + NoBuild: false, + AsyncBuild: false, + } + snaps, _ = snapshot.New(snapconfig, db, sdb.TrieDB(), root) } statedb, _ = state.New(root, sdb, snaps) return snaps, statedb diff --git a/trie/committer.go b/trie/committer.go index 9a7bf48d977fd..90191cf9b1dd5 100644 --- a/trie/committer.go +++ b/trie/committer.go @@ -17,71 +17,65 @@ package trie import ( - "errors" "fmt" - "sync" "github.com/ethereum/go-ethereum/common" ) -// leafChanSize is the size of the leafCh. It's a pretty arbitrary number, to allow -// some parallelism but not incur too much memory overhead. -const leafChanSize = 200 - -// leaf represents a trie leaf value +// leaf represents a trie leaf node type leaf struct { - size int // size of the rlp data (estimate) - hash common.Hash // hash of rlp data - node node // the node to commit + blob []byte // raw blob of leaf + parent common.Hash // the hash of parent node } -// committer is a type used for the trie Commit operation. A committer has some -// internal preallocated temp space, and also a callback that is invoked when -// leaves are committed. The leafs are passed through the `leafCh`, to allow -// some level of parallelism. -// By 'some level' of parallelism, it's still the case that all leaves will be -// processed sequentially - onleaf will never be called in parallel or out of order. +// committer is the tool used for the trie Commit operation. The committer will +// capture all dirty nodes during the commit process and keep them cached in +// insertion order. type committer struct { - onleaf LeafCallback - leafCh chan *leaf -} - -// committers live in a global sync.Pool -var committerPool = sync.Pool{ - New: func() interface{} { - return &committer{} - }, + nodes *NodeSet + tracer *tracer + collectLeaf bool } // newCommitter creates a new committer or picks one from the pool. -func newCommitter() *committer { - return committerPool.Get().(*committer) -} - -func returnCommitterToPool(h *committer) { - h.onleaf = nil - h.leafCh = nil - committerPool.Put(h) +func newCommitter(owner common.Hash, tracer *tracer, collectLeaf bool) *committer { + return &committer{ + nodes: NewNodeSet(owner), + tracer: tracer, + collectLeaf: collectLeaf, + } } -// Commit collapses a node down into a hash node and inserts it into the database -func (c *committer) Commit(n node, db *Database) (hashNode, int, error) { - if db == nil { - return nil, 0, errors.New("no db provided") - } - h, committed, err := c.commit(n, db) +// Commit collapses a node down into a hash node and returns it along with +// the modified nodeset. +func (c *committer) Commit(n node) (hashNode, *NodeSet, error) { + h, err := c.commit(nil, n) if err != nil { - return nil, 0, err + return nil, nil, err } - return h.(hashNode), committed, nil + // Some nodes can be deleted from trie which can't be captured by committer + // itself. Iterate all deleted nodes tracked by tracer and marked them as + // deleted only if they are present in database previously. + for _, path := range c.tracer.deleteList() { + // There are a few possibilities for this scenario(the node is deleted + // but not present in database previously), for example the node was + // embedded in the parent and now deleted from the trie. In this case + // it's noop from database's perspective. + val := c.tracer.getPrev(path) + if len(val) == 0 { + continue + } + c.nodes.markDeleted(path, val) + } + return h.(hashNode), c.nodes, nil } -// commit collapses a node down into a hash node and inserts it into the database -func (c *committer) commit(n node, db *Database) (node, int, error) { +// commit collapses a node down into a hash node and returns it. +func (c *committer) commit(path []byte, n node) (node, error) { // if this path is clean, use available cached data hash, dirty := n.cache() if hash != nil && !dirty { - return hash, 0, nil + return hash, nil } // Commit children, then parent, and remove the dirty flag. switch cn := n.(type) { @@ -91,36 +85,48 @@ func (c *committer) commit(n node, db *Database) (node, int, error) { // If the child is fullNode, recursively commit, // otherwise it can only be hashNode or valueNode. - var childCommitted int if _, ok := cn.Val.(*fullNode); ok { - childV, committed, err := c.commit(cn.Val, db) + childV, err := c.commit(append(path, cn.Key...), cn.Val) if err != nil { - return nil, 0, err + return nil, err } - collapsed.Val, childCommitted = childV, committed + collapsed.Val = childV } - // The key needs to be copied, since we're delivering it to database + // The key needs to be copied, since we're adding it to the + // modified nodeset. collapsed.Key = hexToCompact(cn.Key) - hashedNode := c.store(collapsed, db) + hashedNode := c.store(path, collapsed) if hn, ok := hashedNode.(hashNode); ok { - return hn, childCommitted + 1, nil + return hn, nil } - return collapsed, childCommitted, nil + // The short node now is embedded in its parent. Mark the node as + // deleted if it's present in database previously. It's equivalent + // as deletion from database's perspective. + if prev := c.tracer.getPrev(path); len(prev) != 0 { + c.nodes.markDeleted(path, prev) + } + return collapsed, nil case *fullNode: - hashedKids, childCommitted, err := c.commitChildren(cn, db) + hashedKids, err := c.commitChildren(path, cn) if err != nil { - return nil, 0, err + return nil, err } collapsed := cn.copy() collapsed.Children = hashedKids - hashedNode := c.store(collapsed, db) + hashedNode := c.store(path, collapsed) if hn, ok := hashedNode.(hashNode); ok { - return hn, childCommitted + 1, nil + return hn, nil + } + // The full node now is embedded in its parent. Mark the node as + // deleted if it's present in database previously. It's equivalent + // as deletion from database's perspective. + if prev := c.tracer.getPrev(path); len(prev) != 0 { + c.nodes.markDeleted(path, prev) } - return collapsed, childCommitted, nil + return collapsed, nil case hashNode: - return cn, 0, nil + return cn, nil default: // nil, valuenode shouldn't be committed panic(fmt.Sprintf("%T: invalid node: %v", n, n)) @@ -128,11 +134,8 @@ func (c *committer) commit(n node, db *Database) (node, int, error) { } // commitChildren commits the children of the given fullnode -func (c *committer) commitChildren(n *fullNode, db *Database) ([17]node, int, error) { - var ( - committed int - children [17]node - ) +func (c *committer) commitChildren(path []byte, n *fullNode) ([17]node, error) { + var children [17]node for i := 0; i < 16; i++ { child := n.Children[i] if child == nil { @@ -148,91 +151,62 @@ func (c *committer) commitChildren(n *fullNode, db *Database) ([17]node, int, er // Commit the child recursively and store the "hashed" value. // Note the returned node can be some embedded nodes, so it's // possible the type is not hashNode. - hashed, childCommitted, err := c.commit(child, db) + hashed, err := c.commit(append(path, byte(i)), child) if err != nil { - return children, 0, err + return children, err } children[i] = hashed - committed += childCommitted } // For the 17th child, it's possible the type is valuenode. if n.Children[16] != nil { children[16] = n.Children[16] } - return children, committed, nil + return children, nil } -// store hashes the node n and if we have a storage layer specified, it writes -// the key/value pair to it and tracks any node->child references as well as any -// node->external trie references. -func (c *committer) store(n node, db *Database) node { +// store hashes the node n and adds it to the modified nodeset. If leaf collection +// is enabled, leaf nodes will be tracked in the modified nodeset as well. +func (c *committer) store(path []byte, n node) node { // Larger nodes are replaced by their hash and stored in the database. - var ( - hash, _ = n.cache() - size int - ) + var hash, _ = n.cache() + + // This was not generated - must be a small node stored in the parent. + // In theory, we should check if the node is leaf here (embedded node + // usually is leaf node). But small value (less than 32bytes) is not + // our target (leaves in account trie only). if hash == nil { - // This was not generated - must be a small node stored in the parent. - // In theory, we should apply the leafCall here if it's not nil(embedded - // node usually contains value). But small value(less than 32bytes) is - // not our target. return n - } else { - // We have the hash already, estimate the RLP encoding-size of the node. - // The size is used for mem tracking, does not need to be exact - size = estimateSize(n) } - // If we're using channel-based leaf-reporting, send to channel. - // The leaf channel will be active only when there an active leaf-callback - if c.leafCh != nil { - c.leafCh <- &leaf{ - size: size, - hash: common.BytesToHash(hash), - node: n, + // We have the hash already, estimate the RLP encoding-size of the node. + // The size is used for mem tracking, does not need to be exact + var ( + size = estimateSize(n) + nhash = common.BytesToHash(hash) + mnode = &memoryNode{ + hash: nhash, + node: simplifyNode(n), + size: uint16(size), } - } else if db != nil { - // No leaf-callback used, but there's still a database. Do serial - // insertion - db.lock.Lock() - db.insert(common.BytesToHash(hash), size, n) - db.lock.Unlock() - } - return hash -} - -// commitLoop does the actual insert + leaf callback for nodes. -func (c *committer) commitLoop(db *Database) { - for item := range c.leafCh { - var ( - hash = item.hash - size = item.size - n = item.node - ) - // We are pooling the trie nodes into an intermediate memory cache - db.lock.Lock() - db.insert(hash, size, n) - db.lock.Unlock() - - if c.onleaf != nil { - switch n := n.(type) { - case *shortNode: - if child, ok := n.Val.(valueNode); ok { - c.onleaf(nil, nil, child, hash) - } - case *fullNode: - // For children in range [0, 15], it's impossible - // to contain valueNode. Only check the 17th child. - if n.Children[16] != nil { - c.onleaf(nil, nil, n.Children[16].(valueNode), hash) - } + ) + // Collect the dirty node to nodeset for return. + c.nodes.markUpdated(path, mnode, c.tracer.getPrev(path)) + + // Collect the corresponding leaf node if it's required. We don't check + // full node since it's impossible to store value in fullNode. The key + // length of leaves should be exactly same. + if c.collectLeaf { + if sn, ok := n.(*shortNode); ok { + if val, ok := sn.Val.(valueNode); ok { + c.nodes.addLeaf(&leaf{blob: val, parent: nhash}) } } } + return hash } // estimateSize estimates the size of an rlp-encoded node, without actually // rlp-encoding it (zero allocs). This method has been experimentally tried, and with a trie -// with 1000 leafs, the only errors above 1% are on small shortnodes, where this +// with 1000 leaves, the only errors above 1% are on small shortnodes, where this // method overestimates by 2 or 3 bytes (e.g. 37 instead of 35) func estimateSize(n node) int { switch n := n.(type) { diff --git a/trie/database.go b/trie/database.go index d71abeee476a5..76ca188add9c8 100644 --- a/trie/database.go +++ b/trie/database.go @@ -28,6 +28,7 @@ import ( "github.com/VictoriaMetrics/fastcache" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" @@ -74,8 +75,6 @@ type Database struct { oldest common.Hash // Oldest tracked node, flush-list head newest common.Hash // Newest tracked node, flush-list tail - preimages map[common.Hash][]byte // Preimages of nodes from the secure trie - gctime time.Duration // Time spent on garbage collection since last commit gcnodes uint64 // Nodes garbage collected since last commit gcsize common.StorageSize // Data storage garbage collected since last commit @@ -84,9 +83,9 @@ type Database struct { flushnodes uint64 // Nodes flushed since last commit flushsize common.StorageSize // Data storage flushed since last commit - dirtiesSize common.StorageSize // Storage size of the dirty node cache (exc. metadata) - childrenSize common.StorageSize // Storage size of the external children tracking - preimagesSize common.StorageSize // Storage size of the preimages cache + dirtiesSize common.StorageSize // Storage size of the dirty node cache (exc. metadata) + childrenSize common.StorageSize // Storage size of the external children tracking + preimages *preimageStore // The store for caching preimages lock sync.RWMutex } @@ -164,7 +163,10 @@ func (n *cachedNode) rlp() []byte { // or by regenerating it from the rlp encoded blob. func (n *cachedNode) obj(hash common.Hash) node { if node, ok := n.node.(rawNode); ok { - return mustDecodeNode(hash[:], node) + // The raw-blob format nodes are loaded either from the + // clean cache or the database, they are all in their own + // copy and safe to use unsafe decoder. + return mustDecodeNodeUnsafe(hash[:], node) } return expandNode(hash[:], n.node) } @@ -287,26 +289,22 @@ func NewDatabaseWithConfig(diskdb ethdb.KeyValueStore, config *Config) *Database cleans = fastcache.LoadFromFileOrNew(config.Journal, config.Cache*1024*1024) } } + var preimage *preimageStore + if config != nil && config.Preimages { + preimage = newPreimageStore(diskdb) + } db := &Database{ diskdb: diskdb, cleans: cleans, dirties: map[common.Hash]*cachedNode{{}: { children: make(map[common.Hash]uint16), }}, - } - if config == nil || config.Preimages { // TODO(karalabe): Flip to default off in the future - db.preimages = make(map[common.Hash][]byte) + preimages: preimage, } return db } -// DiskDB retrieves the persistent storage backing the trie database. -func (db *Database) DiskDB() ethdb.KeyValueStore { - return db.diskdb -} - -// insert inserts a collapsed trie node into the memory database. -// The blob size must be specified to allow proper size tracking. +// insert inserts a simplified trie node into the memory database. // All nodes inserted by this function will be reference tracked // and in theory should only used for **trie nodes** insertion. func (db *Database) insert(hash common.Hash, size int, node node) { @@ -318,7 +316,7 @@ func (db *Database) insert(hash common.Hash, size int, node node) { // Create the cached entry for this node entry := &cachedNode{ - node: simplifyNode(node), + node: node, size: uint16(size), flushPrev: db.newest, } @@ -338,24 +336,6 @@ func (db *Database) insert(hash common.Hash, size int, node node) { db.dirtiesSize += common.StorageSize(common.HashLength + entry.size) } -// insertPreimage writes a new trie node pre-image to the memory database if it's -// yet unknown. The method will NOT make a copy of the slice, -// only use if the preimage will NOT be changed later on. -// -// Note, this method assumes that the database's lock is held! -func (db *Database) insertPreimage(hash common.Hash, preimage []byte) { - // Short circuit if preimage collection is disabled - if db.preimages == nil { - return - } - // Track the preimage if a yet unknown one - if _, ok := db.preimages[hash]; ok { - return - } - db.preimages[hash] = preimage - db.preimagesSize += common.StorageSize(common.HashLength + len(preimage)) -} - // node retrieves a cached trie node from memory, or returns nil if none can be // found in the memory cache. func (db *Database) node(hash common.Hash) node { @@ -364,7 +344,10 @@ func (db *Database) node(hash common.Hash) node { if enc := db.cleans.Get(nil, hash[:]); enc != nil { memcacheCleanHitMeter.Mark(1) memcacheCleanReadMeter.Mark(int64(len(enc))) - return mustDecodeNode(hash[:], enc) + + // The returned value from cache is in its own copy, + // safe to use mustDecodeNodeUnsafe for decoding. + return mustDecodeNodeUnsafe(hash[:], enc) } } // Retrieve the node from the dirty cache if available @@ -389,7 +372,9 @@ func (db *Database) node(hash common.Hash) node { memcacheCleanMissMeter.Mark(1) memcacheCleanWriteMeter.Mark(int64(len(enc))) } - return mustDecodeNode(hash[:], enc) + // The returned value from database is in its own copy, + // safe to use mustDecodeNodeUnsafe for decoding. + return mustDecodeNodeUnsafe(hash[:], enc) } // Node retrieves an encoded cached trie node from memory. If it cannot be found @@ -432,24 +417,6 @@ func (db *Database) Node(hash common.Hash) ([]byte, error) { return nil, errors.New("not found") } -// preimage retrieves a cached trie node pre-image from memory. If it cannot be -// found cached, the method queries the persistent database for the content. -func (db *Database) preimage(hash common.Hash) []byte { - // Short circuit if preimage collection is disabled - if db.preimages == nil { - return nil - } - // Retrieve the node from cache if available - db.lock.RLock() - preimage := db.preimages[hash] - db.lock.RUnlock() - - if preimage != nil { - return preimage - } - return rawdb.ReadPreimage(db.diskdb, hash) -} - // Nodes retrieves the hashes of all the nodes cached within the memory database. // This method is extremely expensive and should only be used to validate internal // states in test code. @@ -594,18 +561,9 @@ func (db *Database) Cap(limit common.StorageSize) error { // If the preimage cache got large enough, push to disk. If it's still small // leave for later to deduplicate writes. - flushPreimages := db.preimagesSize > 4*1024*1024 - if flushPreimages { - if db.preimages == nil { - log.Error("Attempted to write preimages whilst disabled") - } else { - rawdb.WritePreimages(batch, db.preimages) - if batch.ValueSize() > ethdb.IdealBatchSize { - if err := batch.Write(); err != nil { - return err - } - batch.Reset() - } + if db.preimages != nil { + if err := db.preimages.commit(false); err != nil { + return err } } // Keep committing nodes from the flush-list until we're below allowance @@ -641,13 +599,6 @@ func (db *Database) Cap(limit common.StorageSize) error { db.lock.Lock() defer db.lock.Unlock() - if flushPreimages { - if db.preimages == nil { - log.Error("Attempted to reset preimage cache whilst disabled") - } else { - db.preimages, db.preimagesSize = make(map[common.Hash][]byte), 0 - } - } for db.oldest != oldest { node := db.dirties[db.oldest] delete(db.dirties, db.oldest) @@ -691,13 +642,9 @@ func (db *Database) Commit(node common.Hash, report bool, callback func(common.H // Move all of the accumulated preimages into a write batch if db.preimages != nil { - rawdb.WritePreimages(batch, db.preimages) - // Since we're going to replay trie node writes into the clean cache, flush out - // any batched pre-images before continuing. - if err := batch.Write(); err != nil { + if err := db.preimages.commit(true); err != nil { return err } - batch.Reset() } // Move the trie itself into the batch, flushing if enough data is accumulated nodes, storage := len(db.dirties), db.dirtiesSize @@ -715,14 +662,12 @@ func (db *Database) Commit(node common.Hash, report bool, callback func(common.H // Uncache any leftovers in the last batch db.lock.Lock() defer db.lock.Unlock() - - batch.Replay(uncacher) + if err := batch.Replay(uncacher); err != nil { + return err + } batch.Reset() // Reset the storage counters and bumped metrics - if db.preimages != nil { - db.preimages, db.preimagesSize = make(map[common.Hash][]byte), 0 - } memcacheCommitTimeTimer.Update(time.Since(start)) memcacheCommitSizeMeter.Mark(int64(storage - db.dirtiesSize)) memcacheCommitNodesMeter.Mark(int64(nodes - len(db.dirties))) @@ -767,9 +712,12 @@ func (db *Database) commit(hash common.Hash, batch ethdb.Batch, uncacher *cleane return err } db.lock.Lock() - batch.Replay(uncacher) + err := batch.Replay(uncacher) batch.Reset() db.lock.Unlock() + if err != nil { + return err + } } return nil } @@ -809,7 +757,7 @@ func (c *cleaner) Put(key []byte, rlp []byte) error { delete(c.db.dirties, hash) c.db.dirtiesSize -= common.StorageSize(common.HashLength + int(node.size)) if node.children != nil { - c.db.dirtiesSize -= common.StorageSize(cachedNodeChildrenSize + len(node.children)*(common.HashLength+2)) + c.db.childrenSize -= common.StorageSize(cachedNodeChildrenSize + len(node.children)*(common.HashLength+2)) } // Move the flushed node into the clean cache to prevent insta-reloads if c.db.cleans != nil { @@ -823,6 +771,54 @@ func (c *cleaner) Delete(key []byte) error { panic("not implemented") } +// Update inserts the dirty nodes in provided nodeset into database and +// link the account trie with multiple storage tries if necessary. +func (db *Database) Update(nodes *MergedNodeSet) error { + db.lock.Lock() + defer db.lock.Unlock() + + // Insert dirty nodes into the database. In the same tree, it must be + // ensured that children are inserted first, then parent so that children + // can be linked with their parent correctly. + // + // Note, the storage tries must be flushed before the account trie to + // retain the invariant that children go into the dirty cache first. + var order []common.Hash + for owner := range nodes.sets { + if owner == (common.Hash{}) { + continue + } + order = append(order, owner) + } + if _, ok := nodes.sets[common.Hash{}]; ok { + order = append(order, common.Hash{}) + } + for _, owner := range order { + subset := nodes.sets[owner] + for _, path := range subset.updates.order { + n, ok := subset.updates.nodes[path] + if !ok { + return fmt.Errorf("missing node %x %v", owner, path) + } + db.insert(n.hash, int(n.size), n.node) + } + } + // Link up the account trie and storage trie if the node points + // to an account trie leaf. + if set, present := nodes.sets[common.Hash{}]; present { + for _, n := range set.leaves { + var account types.StateAccount + if err := rlp.DecodeBytes(n.blob, &account); err != nil { + return err + } + if account.Root != emptyRoot { + db.reference(account.Root, n.parent) + } + } + } + return nil +} + // Size returns the current storage size of the memory cache in front of the // persistent database layer. func (db *Database) Size() (common.StorageSize, common.StorageSize) { @@ -834,7 +830,39 @@ func (db *Database) Size() (common.StorageSize, common.StorageSize) { // counted. var metadataSize = common.StorageSize((len(db.dirties) - 1) * cachedNodeSize) var metarootRefs = common.StorageSize(len(db.dirties[common.Hash{}].children) * (common.HashLength + 2)) - return db.dirtiesSize + db.childrenSize + metadataSize - metarootRefs, db.preimagesSize + var preimageSize common.StorageSize + if db.preimages != nil { + preimageSize = db.preimages.size() + } + return db.dirtiesSize + db.childrenSize + metadataSize - metarootRefs, preimageSize +} + +// GetReader retrieves a node reader belonging to the given state root. +func (db *Database) GetReader(root common.Hash) Reader { + return newHashReader(db) +} + +// hashReader is reader of hashDatabase which implements the Reader interface. +type hashReader struct { + db *Database +} + +// newHashReader initializes the hash reader. +func newHashReader(db *Database) *hashReader { + return &hashReader{db: db} +} + +// Node retrieves the trie node with the given node hash. +// No error will be returned if the node is not found. +func (reader *hashReader) Node(_ common.Hash, _ []byte, hash common.Hash) (node, error) { + return reader.db.node(hash), nil +} + +// NodeBlob retrieves the RLP-encoded trie node blob with the given node hash. +// No error will be returned if the node is not found. +func (reader *hashReader) NodeBlob(_ common.Hash, _ []byte, hash common.Hash) ([]byte, error) { + blob, _ := reader.db.Node(hash) + return blob, nil } // saveCache saves clean state cache to given directory path @@ -876,3 +904,16 @@ func (db *Database) SaveCachePeriodically(dir string, interval time.Duration, st } } } + +// CommitPreimages flushes the dangling preimages to disk. It is meant to be +// called when closing the blockchain object, so that preimages are persisted +// to the database. +func (db *Database) CommitPreimages() error { + db.lock.Lock() + defer db.lock.Unlock() + + if db.preimages == nil { + return nil + } + return db.preimages.commit(true) +} diff --git a/trie/errors.go b/trie/errors.go index 567b80078c062..afe344bed2692 100644 --- a/trie/errors.go +++ b/trie/errors.go @@ -26,10 +26,21 @@ import ( // in the case where a trie node is not present in the local database. It contains // information necessary for retrieving the missing node. type MissingNodeError struct { + Owner common.Hash // owner of the trie if it's 2-layered trie NodeHash common.Hash // hash of the missing node Path []byte // hex-encoded path to the missing node + err error // concrete error for missing trie node +} + +// Unwrap returns the concrete error for missing trie node which +// allows us for further analysis outside. +func (err *MissingNodeError) Unwrap() error { + return err.err } func (err *MissingNodeError) Error() string { - return fmt.Sprintf("missing trie node %x (path %x)", err.NodeHash, err.Path) + if err.Owner == (common.Hash{}) { + return fmt.Sprintf("missing trie node %x (path %x) %v", err.NodeHash, err.Path, err.err) + } + return fmt.Sprintf("missing trie node %x (owner %x) (path %x) %v", err.NodeHash, err.Owner, err.Path, err.err) } diff --git a/trie/hasher.go b/trie/hasher.go index 2949a3ddeece2..e594d6d6b2ae3 100644 --- a/trie/hasher.go +++ b/trie/hasher.go @@ -30,7 +30,7 @@ type hasher struct { sha crypto.KeccakState tmp []byte encbuf rlp.EncoderBuffer - parallel bool // Whether to use paralallel threads when hashing + parallel bool // Whether to use parallel threads when hashing } // hasherPool holds pureHashers @@ -170,8 +170,8 @@ func (h *hasher) fullnodeToHash(n *fullNode, force bool) node { // // All node encoding must be done like this: // -// node.encode(h.encbuf) -// enc := h.encodedBytes() +// node.encode(h.encbuf) +// enc := h.encodedBytes() // // This convention exists because node.encode can only be inlined/escape-analyzed when // called on a concrete receiver type. @@ -191,7 +191,7 @@ func (h *hasher) hashData(data []byte) hashNode { } // proofHash is used to construct trie proofs, and returns the 'collapsed' -// node (for later RLP encoding) aswell as the hashed node -- unless the +// node (for later RLP encoding) as well as the hashed node -- unless the // node is smaller than 32 bytes, in which case it will be returned as is. // This method does not do anything on value- or hash-nodes. func (h *hasher) proofHash(original node) (collapsed, hashed node) { diff --git a/trie/iterator.go b/trie/iterator.go index e0006ee05e3ba..b13651fc04397 100644 --- a/trie/iterator.go +++ b/trie/iterator.go @@ -375,8 +375,12 @@ func (it *nodeIterator) resolveHash(hash hashNode, path []byte) (node, error) { } } } - resolved, err := it.trie.resolveHash(hash, path) - return resolved, err + // Retrieve the specified node from the underlying node reader. + // it.trie.resolveAndTrack is not used since in that function the + // loaded blob will be tracked, while it's not required here since + // all loaded nodes won't be linked to trie at all and track nodes + // may lead to out-of-memory issue. + return it.trie.reader.node(path, common.BytesToHash(hash)) } func (it *nodeIterator) resolveBlob(hash hashNode, path []byte) ([]byte, error) { @@ -385,7 +389,12 @@ func (it *nodeIterator) resolveBlob(hash hashNode, path []byte) ([]byte, error) return blob, nil } } - return it.trie.resolveBlob(hash, path) + // Retrieve the specified node from the underlying node reader. + // it.trie.resolveAndTrack is not used since in that function the + // loaded blob will be tracked, while it's not required here since + // all loaded nodes won't be linked to trie at all and track nodes + // may lead to out-of-memory issue. + return it.trie.reader.nodeBlob(path, common.BytesToHash(hash)) } func (st *nodeIteratorState) resolve(it *nodeIterator, path []byte) error { diff --git a/trie/iterator_test.go b/trie/iterator_test.go index ea8a46bb43018..74b87a25c2330 100644 --- a/trie/iterator_test.go +++ b/trie/iterator_test.go @@ -31,7 +31,7 @@ import ( ) func TestEmptyIterator(t *testing.T) { - trie := newEmpty() + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) iter := trie.NodeIterator(nil) seen := make(map[string]struct{}) @@ -44,7 +44,8 @@ func TestEmptyIterator(t *testing.T) { } func TestIterator(t *testing.T) { - trie := newEmpty() + db := NewDatabase(rawdb.NewMemoryDatabase()) + trie := NewEmpty(db) vals := []struct{ k, v string }{ {"do", "verb"}, {"ether", "wookiedoo"}, @@ -59,8 +60,13 @@ func TestIterator(t *testing.T) { all[val.k] = val.v trie.Update([]byte(val.k), []byte(val.v)) } - trie.Commit(nil) + root, nodes, err := trie.Commit(false) + if err != nil { + t.Fatalf("Failed to commit trie %v", err) + } + db.Update(NewWithNodeSet(nodes)) + trie, _ = New(TrieID(root), db) found := make(map[string]string) it := NewIterator(trie.NodeIterator(nil)) for it.Next() { @@ -80,7 +86,7 @@ type kv struct { } func TestIteratorLargeData(t *testing.T) { - trie := newEmpty() + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) vals := make(map[string]*kv) for i := byte(0); i < 255; i++ { @@ -173,7 +179,7 @@ var testdata2 = []kvs{ } func TestIteratorSeek(t *testing.T) { - trie := newEmpty() + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) for _, val := range testdata1 { trie.Update([]byte(val.k), []byte(val.v)) } @@ -214,17 +220,23 @@ func checkIteratorOrder(want []kvs, it *Iterator) error { } func TestDifferenceIterator(t *testing.T) { - triea := newEmpty() + dba := NewDatabase(rawdb.NewMemoryDatabase()) + triea := NewEmpty(dba) for _, val := range testdata1 { triea.Update([]byte(val.k), []byte(val.v)) } - triea.Commit(nil) + rootA, nodesA, _ := triea.Commit(false) + dba.Update(NewWithNodeSet(nodesA)) + triea, _ = New(TrieID(rootA), dba) - trieb := newEmpty() + dbb := NewDatabase(rawdb.NewMemoryDatabase()) + trieb := NewEmpty(dbb) for _, val := range testdata2 { trieb.Update([]byte(val.k), []byte(val.v)) } - trieb.Commit(nil) + rootB, nodesB, _ := trieb.Commit(false) + dbb.Update(NewWithNodeSet(nodesB)) + trieb, _ = New(TrieID(rootB), dbb) found := make(map[string]string) di, _ := NewDifferenceIterator(triea.NodeIterator(nil), trieb.NodeIterator(nil)) @@ -250,17 +262,23 @@ func TestDifferenceIterator(t *testing.T) { } func TestUnionIterator(t *testing.T) { - triea := newEmpty() + dba := NewDatabase(rawdb.NewMemoryDatabase()) + triea := NewEmpty(dba) for _, val := range testdata1 { triea.Update([]byte(val.k), []byte(val.v)) } - triea.Commit(nil) + rootA, nodesA, _ := triea.Commit(false) + dba.Update(NewWithNodeSet(nodesA)) + triea, _ = New(TrieID(rootA), dba) - trieb := newEmpty() + dbb := NewDatabase(rawdb.NewMemoryDatabase()) + trieb := NewEmpty(dbb) for _, val := range testdata2 { trieb.Update([]byte(val.k), []byte(val.v)) } - trieb.Commit(nil) + rootB, nodesB, _ := trieb.Commit(false) + dbb.Update(NewWithNodeSet(nodesB)) + trieb, _ = New(TrieID(rootB), dbb) di, _ := NewUnionIterator([]NodeIterator{triea.NodeIterator(nil), trieb.NodeIterator(nil)}) it := NewIterator(di) @@ -297,7 +315,7 @@ func TestUnionIterator(t *testing.T) { } func TestIteratorNoDups(t *testing.T) { - tr, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) + tr := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) for _, val := range testdata1 { tr.Update([]byte(val.k), []byte(val.v)) } @@ -312,11 +330,12 @@ func testIteratorContinueAfterError(t *testing.T, memonly bool) { diskdb := memorydb.New() triedb := NewDatabase(diskdb) - tr, _ := New(common.Hash{}, triedb) + tr := NewEmpty(triedb) for _, val := range testdata1 { tr.Update([]byte(val.k), []byte(val.v)) } - tr.Commit(nil) + _, nodes, _ := tr.Commit(false) + triedb.Update(NewWithNodeSet(nodes)) if !memonly { triedb.Commit(tr.Hash(), true, nil) } @@ -337,7 +356,7 @@ func testIteratorContinueAfterError(t *testing.T, memonly bool) { } for i := 0; i < 20; i++ { // Create trie that will load all nodes from DB. - tr, _ := New(tr.Hash(), triedb) + tr, _ := New(TrieID(tr.Hash()), triedb) // Remove a random node from the database. It can't be the root node // because that one is already loaded. @@ -403,11 +422,12 @@ func testIteratorContinueAfterSeekError(t *testing.T, memonly bool) { diskdb := memorydb.New() triedb := NewDatabase(diskdb) - ctr, _ := New(common.Hash{}, triedb) + ctr := NewEmpty(triedb) for _, val := range testdata1 { ctr.Update([]byte(val.k), []byte(val.v)) } - root, _, _ := ctr.Commit(nil) + root, nodes, _ := ctr.Commit(false) + triedb.Update(NewWithNodeSet(nodes)) if !memonly { triedb.Commit(root, true, nil) } @@ -425,7 +445,7 @@ func testIteratorContinueAfterSeekError(t *testing.T, memonly bool) { } // Create a new iterator that seeks to "bars". Seeking can't proceed because // the node is missing. - tr, _ := New(root, triedb) + tr, _ := New(TrieID(root), triedb) it := tr.NodeIterator([]byte("bars")) missing, ok := it.Error().(*MissingNodeError) if !ok { @@ -509,11 +529,11 @@ func (l *loggingDb) Close() error { } // makeLargeTestTrie create a sample test trie -func makeLargeTestTrie() (*Database, *SecureTrie, *loggingDb) { +func makeLargeTestTrie() (*Database, *StateTrie, *loggingDb) { // Create an empty trie logDb := &loggingDb{0, memorydb.New()} triedb := NewDatabase(logDb) - trie, _ := NewSecure(common.Hash{}, triedb) + trie, _ := NewStateTrie(TrieID(common.Hash{}), triedb) // Fill it with some arbitrary data for i := 0; i < 10000; i++ { @@ -525,7 +545,8 @@ func makeLargeTestTrie() (*Database, *SecureTrie, *loggingDb) { val = crypto.Keccak256(val) trie.Update(key, val) } - trie.Commit(nil) + _, nodes, _ := trie.Commit(false) + triedb.Update(NewWithNodeSet(nodes)) // Return the generated trie return triedb, trie, logDb } @@ -546,9 +567,9 @@ func TestNodeIteratorLargeTrie(t *testing.T) { func TestIteratorNodeBlob(t *testing.T) { var ( - db = memorydb.New() - triedb = NewDatabase(db) - trie, _ = New(common.Hash{}, triedb) + db = memorydb.New() + triedb = NewDatabase(db) + trie = NewEmpty(triedb) ) vals := []struct{ k, v string }{ {"do", "verb"}, @@ -564,7 +585,8 @@ func TestIteratorNodeBlob(t *testing.T) { all[val.k] = val.v trie.Update([]byte(val.k), []byte(val.v)) } - trie.Commit(nil) + _, nodes, _ := trie.Commit(false) + triedb.Update(NewWithNodeSet(nodes)) triedb.Cap(0) found := make(map[common.Hash][]byte) diff --git a/trie/node.go b/trie/node.go index bf3f024bb8a78..6ce6551ded8cb 100644 --- a/trie/node.go +++ b/trie/node.go @@ -99,6 +99,7 @@ func (n valueNode) fstring(ind string) string { return fmt.Sprintf("%x ", []byte(n)) } +// mustDecodeNode is a wrapper of decodeNode and panic if any error is encountered. func mustDecodeNode(hash, buf []byte) node { n, err := decodeNode(hash, buf) if err != nil { @@ -107,8 +108,29 @@ func mustDecodeNode(hash, buf []byte) node { return n } -// decodeNode parses the RLP encoding of a trie node. +// mustDecodeNodeUnsafe is a wrapper of decodeNodeUnsafe and panic if any error is +// encountered. +func mustDecodeNodeUnsafe(hash, buf []byte) node { + n, err := decodeNodeUnsafe(hash, buf) + if err != nil { + panic(fmt.Sprintf("node %x: %v", hash, err)) + } + return n +} + +// decodeNode parses the RLP encoding of a trie node. It will deep-copy the passed +// byte slice for decoding, so it's safe to modify the byte slice afterwards. The- +// decode performance of this function is not optimal, but it is suitable for most +// scenarios with low performance requirements and hard to determine whether the +// byte slice be modified or not. func decodeNode(hash, buf []byte) (node, error) { + return decodeNodeUnsafe(hash, common.CopyBytes(buf)) +} + +// decodeNodeUnsafe parses the RLP encoding of a trie node. The passed byte slice +// will be directly referenced by node without bytes deep copy, so the input MUST +// not be changed after. +func decodeNodeUnsafe(hash, buf []byte) (node, error) { if len(buf) == 0 { return nil, io.ErrUnexpectedEOF } @@ -141,7 +163,7 @@ func decodeShort(hash, elems []byte) (node, error) { if err != nil { return nil, fmt.Errorf("invalid value node: %v", err) } - return &shortNode{key, append(valueNode{}, val...), flag}, nil + return &shortNode{key, valueNode(val), flag}, nil } r, _, err := decodeRef(rest) if err != nil { @@ -164,7 +186,7 @@ func decodeFull(hash, elems []byte) (*fullNode, error) { return n, err } if len(val) > 0 { - n.Children[16] = append(valueNode{}, val...) + n.Children[16] = valueNode(val) } return n, nil } @@ -190,7 +212,7 @@ func decodeRef(buf []byte) (node, []byte, error) { // empty node return nil, rest, nil case kind == rlp.String && len(val) == 32: - return append(hashNode{}, val...), rest, nil + return hashNode(val), rest, nil default: return nil, nil, fmt.Errorf("invalid RLP string size %d (want 0 or 32)", len(val)) } diff --git a/trie/node_test.go b/trie/node_test.go index ac1d8fbef3e6b..9b8b33748fa7e 100644 --- a/trie/node_test.go +++ b/trie/node_test.go @@ -20,6 +20,7 @@ import ( "bytes" "testing" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/rlp" ) @@ -92,3 +93,123 @@ func TestDecodeFullNode(t *testing.T) { t.Fatalf("decode full node err: %v", err) } } + +// goos: darwin +// goarch: arm64 +// pkg: github.com/ethereum/go-ethereum/trie +// BenchmarkEncodeShortNode +// BenchmarkEncodeShortNode-8 16878850 70.81 ns/op 48 B/op 1 allocs/op +func BenchmarkEncodeShortNode(b *testing.B) { + node := &shortNode{ + Key: []byte{0x1, 0x2}, + Val: hashNode(randBytes(32)), + } + b.ResetTimer() + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + nodeToBytes(node) + } +} + +// goos: darwin +// goarch: arm64 +// pkg: github.com/ethereum/go-ethereum/trie +// BenchmarkEncodeFullNode +// BenchmarkEncodeFullNode-8 4323273 284.4 ns/op 576 B/op 1 allocs/op +func BenchmarkEncodeFullNode(b *testing.B) { + node := &fullNode{} + for i := 0; i < 16; i++ { + node.Children[i] = hashNode(randBytes(32)) + } + b.ResetTimer() + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + nodeToBytes(node) + } +} + +// goos: darwin +// goarch: arm64 +// pkg: github.com/ethereum/go-ethereum/trie +// BenchmarkDecodeShortNode +// BenchmarkDecodeShortNode-8 7925638 151.0 ns/op 157 B/op 4 allocs/op +func BenchmarkDecodeShortNode(b *testing.B) { + node := &shortNode{ + Key: []byte{0x1, 0x2}, + Val: hashNode(randBytes(32)), + } + blob := nodeToBytes(node) + hash := crypto.Keccak256(blob) + + b.ResetTimer() + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + mustDecodeNode(hash, blob) + } +} + +// goos: darwin +// goarch: arm64 +// pkg: github.com/ethereum/go-ethereum/trie +// BenchmarkDecodeShortNodeUnsafe +// BenchmarkDecodeShortNodeUnsafe-8 9027476 128.6 ns/op 109 B/op 3 allocs/op +func BenchmarkDecodeShortNodeUnsafe(b *testing.B) { + node := &shortNode{ + Key: []byte{0x1, 0x2}, + Val: hashNode(randBytes(32)), + } + blob := nodeToBytes(node) + hash := crypto.Keccak256(blob) + + b.ResetTimer() + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + mustDecodeNodeUnsafe(hash, blob) + } +} + +// goos: darwin +// goarch: arm64 +// pkg: github.com/ethereum/go-ethereum/trie +// BenchmarkDecodeFullNode +// BenchmarkDecodeFullNode-8 1597462 761.9 ns/op 1280 B/op 18 allocs/op +func BenchmarkDecodeFullNode(b *testing.B) { + node := &fullNode{} + for i := 0; i < 16; i++ { + node.Children[i] = hashNode(randBytes(32)) + } + blob := nodeToBytes(node) + hash := crypto.Keccak256(blob) + + b.ResetTimer() + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + mustDecodeNode(hash, blob) + } +} + +// goos: darwin +// goarch: arm64 +// pkg: github.com/ethereum/go-ethereum/trie +// BenchmarkDecodeFullNodeUnsafe +// BenchmarkDecodeFullNodeUnsafe-8 1789070 687.1 ns/op 704 B/op 17 allocs/op +func BenchmarkDecodeFullNodeUnsafe(b *testing.B) { + node := &fullNode{} + for i := 0; i < 16; i++ { + node.Children[i] = hashNode(randBytes(32)) + } + blob := nodeToBytes(node) + hash := crypto.Keccak256(blob) + + b.ResetTimer() + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + mustDecodeNodeUnsafe(hash, blob) + } +} diff --git a/trie/nodeset.go b/trie/nodeset.go new file mode 100644 index 0000000000000..0f9d4ea01570b --- /dev/null +++ b/trie/nodeset.go @@ -0,0 +1,209 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package trie + +import ( + "fmt" + "reflect" + "strings" + + "github.com/ethereum/go-ethereum/common" +) + +// memoryNode is all the information we know about a single cached trie node +// in the memory. +type memoryNode struct { + hash common.Hash // Node hash, computed by hashing rlp value, empty for deleted nodes + size uint16 // Byte size of the useful cached data, 0 for deleted nodes + node node // Cached collapsed trie node, or raw rlp data, nil for deleted nodes +} + +// memoryNodeSize is the raw size of a memoryNode data structure without any +// node data included. It's an approximate size, but should be a lot better +// than not counting them. +// nolint:unused +var memoryNodeSize = int(reflect.TypeOf(memoryNode{}).Size()) + +// memorySize returns the total memory size used by this node. +// nolint:unused +func (n *memoryNode) memorySize(key int) int { + return int(n.size) + memoryNodeSize + key +} + +// rlp returns the raw rlp encoded blob of the cached trie node, either directly +// from the cache, or by regenerating it from the collapsed node. +// nolint:unused +func (n *memoryNode) rlp() []byte { + if node, ok := n.node.(rawNode); ok { + return node + } + return nodeToBytes(n.node) +} + +// obj returns the decoded and expanded trie node, either directly from the cache, +// or by regenerating it from the rlp encoded blob. +// nolint:unused +func (n *memoryNode) obj() node { + if node, ok := n.node.(rawNode); ok { + return mustDecodeNode(n.hash[:], node) + } + return expandNode(n.hash[:], n.node) +} + +// nodeWithPrev wraps the memoryNode with the previous node value. +type nodeWithPrev struct { + *memoryNode + prev []byte // RLP-encoded previous value, nil means it's non-existent +} + +// unwrap returns the internal memoryNode object. +// nolint:unused +func (n *nodeWithPrev) unwrap() *memoryNode { + return n.memoryNode +} + +// memorySize returns the total memory size used by this node. It overloads +// the function in memoryNode by counting the size of previous value as well. +// nolint: unused +func (n *nodeWithPrev) memorySize(key int) int { + return n.memoryNode.memorySize(key) + len(n.prev) +} + +// nodesWithOrder represents a collection of dirty nodes which includes +// newly-inserted and updated nodes. The modification order of all nodes +// is represented by order list. +type nodesWithOrder struct { + order []string // the path list of dirty nodes, sort by insertion order + nodes map[string]*nodeWithPrev // the map of dirty nodes, keyed by node path +} + +// NodeSet contains all dirty nodes collected during the commit operation. +// Each node is keyed by path. It's not thread-safe to use. +type NodeSet struct { + owner common.Hash // the identifier of the trie + updates *nodesWithOrder // the set of updated nodes(newly inserted, updated) + deletes map[string][]byte // the map of deleted nodes, keyed by node + leaves []*leaf // the list of dirty leaves +} + +// NewNodeSet initializes an empty node set to be used for tracking dirty nodes +// from a specific account or storage trie. The owner is zero for the account +// trie and the owning account address hash for storage tries. +func NewNodeSet(owner common.Hash) *NodeSet { + return &NodeSet{ + owner: owner, + updates: &nodesWithOrder{ + nodes: make(map[string]*nodeWithPrev), + }, + deletes: make(map[string][]byte), + } +} + +// NewNodeSetWithDeletion initializes the nodeset with provided deletion set. +func NewNodeSetWithDeletion(owner common.Hash, paths [][]byte, prev [][]byte) *NodeSet { + set := NewNodeSet(owner) + for i, path := range paths { + set.markDeleted(path, prev[i]) + } + return set +} + +// markUpdated marks the node as dirty(newly-inserted or updated) with provided +// node path, node object along with its previous value. +func (set *NodeSet) markUpdated(path []byte, node *memoryNode, prev []byte) { + set.updates.order = append(set.updates.order, string(path)) + set.updates.nodes[string(path)] = &nodeWithPrev{ + memoryNode: node, + prev: prev, + } +} + +// markDeleted marks the node as deleted with provided path and previous value. +func (set *NodeSet) markDeleted(path []byte, prev []byte) { + set.deletes[string(path)] = prev +} + +// addLeaf collects the provided leaf node into set. +func (set *NodeSet) addLeaf(node *leaf) { + set.leaves = append(set.leaves, node) +} + +// Size returns the number of updated and deleted nodes contained in the set. +func (set *NodeSet) Size() (int, int) { + return len(set.updates.order), len(set.deletes) +} + +// Hashes returns the hashes of all updated nodes. TODO(rjl493456442) how can +// we get rid of it? +func (set *NodeSet) Hashes() []common.Hash { + var ret []common.Hash + for _, node := range set.updates.nodes { + ret = append(ret, node.hash) + } + return ret +} + +// Summary returns a string-representation of the NodeSet. +func (set *NodeSet) Summary() string { + var out = new(strings.Builder) + fmt.Fprintf(out, "nodeset owner: %v\n", set.owner) + if set.updates != nil { + for _, key := range set.updates.order { + updated := set.updates.nodes[key] + if updated.prev != nil { + fmt.Fprintf(out, " [*]: %x -> %v prev: %x\n", key, updated.hash, updated.prev) + } else { + fmt.Fprintf(out, " [+]: %x -> %v\n", key, updated.hash) + } + } + } + for k, n := range set.deletes { + fmt.Fprintf(out, " [-]: %x -> %x\n", k, n) + } + for _, n := range set.leaves { + fmt.Fprintf(out, "[leaf]: %v\n", n) + } + return out.String() +} + +// MergedNodeSet represents a merged dirty node set for a group of tries. +type MergedNodeSet struct { + sets map[common.Hash]*NodeSet +} + +// NewMergedNodeSet initializes an empty merged set. +func NewMergedNodeSet() *MergedNodeSet { + return &MergedNodeSet{sets: make(map[common.Hash]*NodeSet)} +} + +// NewWithNodeSet constructs a merged nodeset with the provided single set. +func NewWithNodeSet(set *NodeSet) *MergedNodeSet { + merged := NewMergedNodeSet() + merged.Merge(set) + return merged +} + +// Merge merges the provided dirty nodes of a trie into the set. The assumption +// is held that no duplicated set belonging to the same trie will be merged twice. +func (set *MergedNodeSet) Merge(other *NodeSet) error { + _, present := set.sets[other.owner] + if present { + return fmt.Errorf("duplicate trie for owner %#x", other.owner) + } + set.sets[other.owner] = other + return nil +} diff --git a/trie/preimages.go b/trie/preimages.go new file mode 100644 index 0000000000000..66f34117c1e83 --- /dev/null +++ b/trie/preimages.go @@ -0,0 +1,95 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package trie + +import ( + "sync" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" +) + +// preimageStore is the store for caching preimages of node key. +type preimageStore struct { + lock sync.RWMutex + disk ethdb.KeyValueStore + preimages map[common.Hash][]byte // Preimages of nodes from the secure trie + preimagesSize common.StorageSize // Storage size of the preimages cache +} + +// newPreimageStore initializes the store for caching preimages. +func newPreimageStore(disk ethdb.KeyValueStore) *preimageStore { + return &preimageStore{ + disk: disk, + preimages: make(map[common.Hash][]byte), + } +} + +// insertPreimage writes a new trie node pre-image to the memory database if it's +// yet unknown. The method will NOT make a copy of the slice, only use if the +// preimage will NOT be changed later on. +func (store *preimageStore) insertPreimage(preimages map[common.Hash][]byte) { + store.lock.Lock() + defer store.lock.Unlock() + + for hash, preimage := range preimages { + if _, ok := store.preimages[hash]; ok { + continue + } + store.preimages[hash] = preimage + store.preimagesSize += common.StorageSize(common.HashLength + len(preimage)) + } +} + +// preimage retrieves a cached trie node pre-image from memory. If it cannot be +// found cached, the method queries the persistent database for the content. +func (store *preimageStore) preimage(hash common.Hash) []byte { + store.lock.RLock() + preimage := store.preimages[hash] + store.lock.RUnlock() + + if preimage != nil { + return preimage + } + return rawdb.ReadPreimage(store.disk, hash) +} + +// commit flushes the cached preimages into the disk. +func (store *preimageStore) commit(force bool) error { + store.lock.Lock() + defer store.lock.Unlock() + + if store.preimagesSize <= 4*1024*1024 && !force { + return nil + } + batch := store.disk.NewBatch() + rawdb.WritePreimages(batch, store.preimages) + if err := batch.Write(); err != nil { + return err + } + store.preimages, store.preimagesSize = make(map[common.Hash][]byte), 0 + return nil +} + +// size returns the current storage size of accumulated preimages. +func (store *preimageStore) size() common.StorageSize { + store.lock.RLock() + defer store.lock.RUnlock() + + return store.preimagesSize +} diff --git a/trie/proof.go b/trie/proof.go index f42dcc761bee7..8e706f886b59c 100644 --- a/trie/proof.go +++ b/trie/proof.go @@ -35,9 +35,12 @@ import ( // with the node that proves the absence of the key. func (t *Trie) Prove(key []byte, fromLevel uint, proofDb ethdb.KeyValueWriter) error { // Collect all nodes on the path to key. + var ( + prefix []byte + nodes []node + tn = t.root + ) key = keybytesToHex(key) - var nodes []node - tn := t.root for len(key) > 0 && tn != nil { switch n := tn.(type) { case *shortNode: @@ -46,18 +49,25 @@ func (t *Trie) Prove(key []byte, fromLevel uint, proofDb ethdb.KeyValueWriter) e tn = nil } else { tn = n.Val + prefix = append(prefix, n.Key...) key = key[len(n.Key):] } nodes = append(nodes, n) case *fullNode: tn = n.Children[key[0]] + prefix = append(prefix, key[0]) key = key[1:] nodes = append(nodes, n) case hashNode: + // Retrieve the specified node from the underlying node reader. + // trie.resolveAndTrack is not used since in that function the + // loaded blob will be tracked, while it's not required here since + // all loaded nodes won't be linked to trie at all and track nodes + // may lead to out-of-memory issue. var err error - tn, err = t.resolveHash(n, nil) + tn, err = t.reader.node(prefix, common.BytesToHash(n)) if err != nil { - log.Error(fmt.Sprintf("Unhandled trie error: %v", err)) + log.Error("Unhandled trie error in Trie.Prove", "err", err) return err } default: @@ -94,7 +104,7 @@ func (t *Trie) Prove(key []byte, fromLevel uint, proofDb ethdb.KeyValueWriter) e // If the trie does not contain a value for key, the returned proof contains all // nodes of the longest existing prefix of the key (at least the root node), ending // with the node that proves the absence of the key. -func (t *SecureTrie) Prove(key []byte, fromLevel uint, proofDb ethdb.KeyValueWriter) error { +func (t *StateTrie) Prove(key []byte, fromLevel uint, proofDb ethdb.KeyValueWriter) error { return t.trie.Prove(key, fromLevel, proofDb) } @@ -333,9 +343,9 @@ findFork: // unset removes all internal node references either the left most or right most. // It can meet these scenarios: // -// - The given path is existent in the trie, unset the associated nodes with the -// specific direction -// - The given path is non-existent in the trie +// - The given path is existent in the trie, unset the associated nodes with the +// specific direction +// - The given path is non-existent in the trie // - the fork point is a fullnode, the corresponding child pointed by path // is nil, return // - the fork point is a shortnode, the shortnode is included in the range, @@ -367,11 +377,12 @@ func unset(parent node, child node, key []byte, pos int, removeLeft bool) error // branch. The parent must be a fullnode. fn := parent.(*fullNode) fn.Children[key[pos-1]] = nil - } else { - // The key of fork shortnode is greater than the - // path(it doesn't belong to the range), keep - // it with the cached hash available. } + //else { + // The key of fork shortnode is greater than the + // path(it doesn't belong to the range), keep + // it with the cached hash available. + //} } else { if bytes.Compare(cld.Key, key[pos:]) > 0 { // The key of fork shortnode is greater than the @@ -379,11 +390,12 @@ func unset(parent node, child node, key []byte, pos int, removeLeft bool) error // branch. The parent must be a fullnode. fn := parent.(*fullNode) fn.Children[key[pos-1]] = nil - } else { - // The key of fork shortnode is less than the - // path(it doesn't belong to the range), keep - // it with the cached hash available. } + //else { + // The key of fork shortnode is less than the + // path(it doesn't belong to the range), keep + // it with the cached hash available. + //} } return nil } @@ -450,15 +462,15 @@ func hasRightElement(node node, key []byte) bool { // Expect the normal case, this function can also be used to verify the following // range proofs: // -// - All elements proof. In this case the proof can be nil, but the range should -// be all the leaves in the trie. +// - All elements proof. In this case the proof can be nil, but the range should +// be all the leaves in the trie. // -// - One element proof. In this case no matter the edge proof is a non-existent -// proof or not, we can always verify the correctness of the proof. +// - One element proof. In this case no matter the edge proof is a non-existent +// proof or not, we can always verify the correctness of the proof. // -// - Zero element proof. In this case a single non-existent proof is enough to prove. -// Besides, if there are still some other leaves available on the right side, then -// an error will be returned. +// - Zero element proof. In this case a single non-existent proof is enough to prove. +// Besides, if there are still some other leaves available on the right side, then +// an error will be returned. // // Except returning the error to indicate the proof is valid or not, the function will // also return a flag to indicate whether there exists more accounts/slots in the trie. @@ -551,7 +563,7 @@ func VerifyRangeProof(rootHash common.Hash, firstKey []byte, lastKey []byte, key } // Rebuild the trie with the leaf stream, the shape of trie // should be same with the original one. - tr := newWithRootNode(root) + tr := &Trie{root: root, reader: newEmptyReader()} if empty { tr.root = nil } diff --git a/trie/proof_test.go b/trie/proof_test.go index cdf5cf6050980..61667b20ab132 100644 --- a/trie/proof_test.go +++ b/trie/proof_test.go @@ -80,7 +80,7 @@ func TestProof(t *testing.T) { } func TestOneElementProof(t *testing.T) { - trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) updateString(trie, "k", "v") for i, prover := range makeProvers(trie) { proof := prover([]byte("k")) @@ -131,7 +131,7 @@ func TestBadProof(t *testing.T) { // Tests that missing keys can also be proven. The test explicitly uses a single // entry trie and checks for missing keys both before and after the single entry. func TestMissingKeyProof(t *testing.T) { - trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) updateString(trie, "k", "v") for i, key := range []string{"a", "j", "l", "z"} { @@ -205,7 +205,7 @@ func TestRangeProofWithNonExistentProof(t *testing.T) { proof := memorydb.New() // Short circuit if the decreased key is same with the previous key - first := decreseKey(common.CopyBytes(entries[start].k)) + first := decreaseKey(common.CopyBytes(entries[start].k)) if start != 0 && bytes.Equal(first, entries[start-1].k) { continue } @@ -214,7 +214,7 @@ func TestRangeProofWithNonExistentProof(t *testing.T) { continue } // Short circuit if the increased key is same with the next key - last := increseKey(common.CopyBytes(entries[end-1].k)) + last := increaseKey(common.CopyBytes(entries[end-1].k)) if end != len(entries) && bytes.Equal(last, entries[end].k) { continue } @@ -274,7 +274,7 @@ func TestRangeProofWithInvalidNonExistentProof(t *testing.T) { // Case 1 start, end := 100, 200 - first := decreseKey(common.CopyBytes(entries[start].k)) + first := decreaseKey(common.CopyBytes(entries[start].k)) proof := memorydb.New() if err := trie.Prove(first, 0, proof); err != nil { @@ -297,7 +297,7 @@ func TestRangeProofWithInvalidNonExistentProof(t *testing.T) { // Case 2 start, end = 100, 200 - last := increseKey(common.CopyBytes(entries[end-1].k)) + last := increaseKey(common.CopyBytes(entries[end-1].k)) proof = memorydb.New() if err := trie.Prove(entries[start].k, 0, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) @@ -343,7 +343,7 @@ func TestOneElementRangeProof(t *testing.T) { // One element with left non-existent edge proof start = 1000 - first := decreseKey(common.CopyBytes(entries[start].k)) + first := decreaseKey(common.CopyBytes(entries[start].k)) proof = memorydb.New() if err := trie.Prove(first, 0, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) @@ -358,7 +358,7 @@ func TestOneElementRangeProof(t *testing.T) { // One element with right non-existent edge proof start = 1000 - last := increseKey(common.CopyBytes(entries[start].k)) + last := increaseKey(common.CopyBytes(entries[start].k)) proof = memorydb.New() if err := trie.Prove(entries[start].k, 0, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) @@ -373,7 +373,7 @@ func TestOneElementRangeProof(t *testing.T) { // One element with two non-existent edge proofs start = 1000 - first, last = decreseKey(common.CopyBytes(entries[start].k)), increseKey(common.CopyBytes(entries[start].k)) + first, last = decreaseKey(common.CopyBytes(entries[start].k)), increaseKey(common.CopyBytes(entries[start].k)) proof = memorydb.New() if err := trie.Prove(first, 0, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) @@ -387,7 +387,7 @@ func TestOneElementRangeProof(t *testing.T) { } // Test the mini trie with only a single element. - tinyTrie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) + tinyTrie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) entry := &kv{randBytes(32), randBytes(20), false} tinyTrie.Update(entry.k, entry.v) @@ -459,7 +459,7 @@ func TestAllElementsProof(t *testing.T) { // TestSingleSideRangeProof tests the range starts from zero. func TestSingleSideRangeProof(t *testing.T) { for i := 0; i < 64; i++ { - trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) var entries entrySlice for i := 0; i < 4096; i++ { value := &kv{randBytes(32), randBytes(20), false} @@ -494,7 +494,7 @@ func TestSingleSideRangeProof(t *testing.T) { // TestReverseSingleSideRangeProof tests the range ends with 0xffff...fff. func TestReverseSingleSideRangeProof(t *testing.T) { for i := 0; i < 64; i++ { - trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) var entries entrySlice for i := 0; i < 4096; i++ { value := &kv{randBytes(32), randBytes(20), false} @@ -601,7 +601,7 @@ func TestBadRangeProof(t *testing.T) { // TestGappedRangeProof focuses on the small trie with embedded nodes. // If the gapped node is embedded in the trie, it should be detected too. func TestGappedRangeProof(t *testing.T) { - trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) var entries []*kv // Sorted entries for i := byte(0); i < 10; i++ { value := &kv{common.LeftPadBytes([]byte{i}, 32), []byte{i}, false} @@ -641,9 +641,9 @@ func TestSameSideProofs(t *testing.T) { sort.Sort(entries) pos := 1000 - first := decreseKey(common.CopyBytes(entries[pos].k)) - first = decreseKey(first) - last := decreseKey(common.CopyBytes(entries[pos].k)) + first := decreaseKey(common.CopyBytes(entries[pos].k)) + first = decreaseKey(first) + last := decreaseKey(common.CopyBytes(entries[pos].k)) proof := memorydb.New() if err := trie.Prove(first, 0, proof); err != nil { @@ -657,9 +657,9 @@ func TestSameSideProofs(t *testing.T) { t.Fatalf("Expected error, got nil") } - first = increseKey(common.CopyBytes(entries[pos].k)) - last = increseKey(common.CopyBytes(entries[pos].k)) - last = increseKey(last) + first = increaseKey(common.CopyBytes(entries[pos].k)) + last = increaseKey(common.CopyBytes(entries[pos].k)) + last = increaseKey(last) proof = memorydb.New() if err := trie.Prove(first, 0, proof); err != nil { @@ -675,7 +675,7 @@ func TestSameSideProofs(t *testing.T) { } func TestHasRightElement(t *testing.T) { - trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) var entries entrySlice for i := 0; i < 4096; i++ { value := &kv{randBytes(32), randBytes(20), false} @@ -765,7 +765,7 @@ func TestEmptyRangeProof(t *testing.T) { } for _, c := range cases { proof := memorydb.New() - first := increseKey(common.CopyBytes(entries[c.pos].k)) + first := increaseKey(common.CopyBytes(entries[c.pos].k)) if err := trie.Prove(first, 0, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) } @@ -904,7 +904,7 @@ func mutateByte(b []byte) { } } -func increseKey(key []byte) []byte { +func increaseKey(key []byte) []byte { for i := len(key) - 1; i >= 0; i-- { key[i]++ if key[i] != 0x0 { @@ -914,7 +914,7 @@ func increseKey(key []byte) []byte { return key } -func decreseKey(key []byte) []byte { +func decreaseKey(key []byte) []byte { for i := len(key) - 1; i >= 0; i-- { key[i]-- if key[i] != 0xff { @@ -1028,7 +1028,7 @@ func benchmarkVerifyRangeNoProof(b *testing.B, size int) { } func randomTrie(n int) (*Trie, map[string]*kv) { - trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) vals := make(map[string]*kv) for i := byte(0); i < 100; i++ { value := &kv{common.LeftPadBytes([]byte{i}, 32), []byte{i}, false} @@ -1053,7 +1053,7 @@ func randBytes(n int) []byte { } func nonRandomTrie(n int) (*Trie, map[string]*kv) { - trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) vals := make(map[string]*kv) max := uint64(0xffffffffffffffff) for i := uint64(0); i < uint64(n); i++ { @@ -1078,7 +1078,7 @@ func TestRangeProofKeysWithSharedPrefix(t *testing.T) { common.Hex2Bytes("02"), common.Hex2Bytes("03"), } - trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) for i, key := range keys { trie.Update(key, vals[i]) } diff --git a/trie/secure_trie.go b/trie/secure_trie.go index 248b93544d2fe..96faab1582651 100644 --- a/trie/secure_trie.go +++ b/trie/secure_trie.go @@ -17,89 +17,111 @@ package trie import ( - "fmt" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" ) -// SecureTrie wraps a trie with key hashing. In a secure trie, all +// SecureTrie is the old name of StateTrie. +// Deprecated: use StateTrie. +type SecureTrie = StateTrie + +// NewSecure creates a new StateTrie. +// Deprecated: use NewStateTrie. +func NewSecure(stateRoot common.Hash, owner common.Hash, root common.Hash, db *Database) (*SecureTrie, error) { + id := &ID{ + StateRoot: stateRoot, + Owner: owner, + Root: root, + } + return NewStateTrie(id, db) +} + +// StateTrie wraps a trie with key hashing. In a stateTrie trie, all // access operations hash the key using keccak256. This prevents // calling code from creating long chains of nodes that // increase the access time. // -// Contrary to a regular trie, a SecureTrie can only be created with +// Contrary to a regular trie, a StateTrie can only be created with // New and must have an attached database. The database also stores -// the preimage of each key. +// the preimage of each key if preimage recording is enabled. // -// SecureTrie is not safe for concurrent use. -type SecureTrie struct { +// StateTrie is not safe for concurrent use. +type StateTrie struct { trie Trie + preimages *preimageStore hashKeyBuf [common.HashLength]byte secKeyCache map[string][]byte - secKeyCacheOwner *SecureTrie // Pointer to self, replace the key cache on mismatch + secKeyCacheOwner *StateTrie // Pointer to self, replace the key cache on mismatch } -// NewSecure creates a trie with an existing root node from a backing database -// and optional intermediate in-memory node pool. +// NewStateTrie creates a trie with an existing root node from a backing database. // // If root is the zero hash or the sha3 hash of an empty string, the // trie is initially empty. Otherwise, New will panic if db is nil // and returns MissingNodeError if the root node cannot be found. -// -// Accessing the trie loads nodes from the database or node pool on demand. -// Loaded nodes are kept around until their 'cache generation' expires. -// A new cache generation is created by each call to Commit. -// cachelimit sets the number of past cache generations to keep. -func NewSecure(root common.Hash, db *Database) (*SecureTrie, error) { +func NewStateTrie(id *ID, db *Database) (*StateTrie, error) { if db == nil { - panic("trie.NewSecure called without a database") + panic("trie.NewStateTrie called without a database") } - trie, err := New(root, db) + trie, err := New(id, db) if err != nil { return nil, err } - return &SecureTrie{trie: *trie}, nil + return &StateTrie{trie: *trie, preimages: db.preimages}, nil } // Get returns the value for key stored in the trie. // The value bytes must not be modified by the caller. -func (t *SecureTrie) Get(key []byte) []byte { +func (t *StateTrie) Get(key []byte) []byte { res, err := t.TryGet(key) if err != nil { - log.Error(fmt.Sprintf("Unhandled trie error: %v", err)) + log.Error("Unhandled trie error in StateTrie.Get", "err", err) } return res } // TryGet returns the value for key stored in the trie. // The value bytes must not be modified by the caller. -// If a node was not found in the database, a MissingNodeError is returned. -func (t *SecureTrie) TryGet(key []byte) ([]byte, error) { +// If the specified node is not in the trie, nil will be returned. +// If a trie node is not found in the database, a MissingNodeError is returned. +func (t *StateTrie) TryGet(key []byte) ([]byte, error) { return t.trie.TryGet(t.hashKey(key)) } -// TryGetNode attempts to retrieve a trie node by compact-encoded path. It is not -// possible to use keybyte-encoding as the path might contain odd nibbles. -func (t *SecureTrie) TryGetNode(path []byte) ([]byte, int, error) { - return t.trie.TryGetNode(path) +// TryGetAccount attempts to retrieve an account with provided trie path. +// If the specified account is not in the trie, nil will be returned. +// If a trie node is not found in the database, a MissingNodeError is returned. +func (t *StateTrie) TryGetAccount(key []byte) (*types.StateAccount, error) { + res, err := t.trie.TryGet(t.hashKey(key)) + if res == nil || err != nil { + return nil, err + } + ret := new(types.StateAccount) + err = rlp.DecodeBytes(res, ret) + return ret, err } -// TryUpdateAccount account will abstract the write of an account to the -// secure trie. -func (t *SecureTrie) TryUpdateAccount(key []byte, acc *types.StateAccount) error { - hk := t.hashKey(key) - data, err := rlp.EncodeToBytes(acc) - if err != nil { - return err - } - if err := t.trie.TryUpdate(hk, data); err != nil { - return err +// TryGetAccountWithPreHashedKey does the same thing as TryGetAccount, however +// it expects a key that is already hashed. This constitutes an abstraction leak, +// since the client code needs to know the key format. +func (t *StateTrie) TryGetAccountWithPreHashedKey(key []byte) (*types.StateAccount, error) { + res, err := t.trie.TryGet(key) + if res == nil || err != nil { + return nil, err } - t.getSecKeyCache()[string(hk)] = common.CopyBytes(key) - return nil + ret := new(types.StateAccount) + err = rlp.DecodeBytes(res, ret) + return ret, err +} + +// TryGetNode attempts to retrieve a trie node by compact-encoded path. It is not +// possible to use keybyte-encoding as the path might contain odd nibbles. +// If the specified trie node is not in the trie, nil will be returned. +// If a trie node is not found in the database, a MissingNodeError is returned. +func (t *StateTrie) TryGetNode(path []byte) ([]byte, int, error) { + return t.trie.TryGetNode(path) } // Update associates key with value in the trie. Subsequent calls to @@ -108,9 +130,9 @@ func (t *SecureTrie) TryUpdateAccount(key []byte, acc *types.StateAccount) error // // The value bytes must not be modified by the caller while they are // stored in the trie. -func (t *SecureTrie) Update(key, value []byte) { +func (t *StateTrie) Update(key, value []byte) { if err := t.TryUpdate(key, value); err != nil { - log.Error(fmt.Sprintf("Unhandled trie error: %v", err)) + log.Error("Unhandled trie error in StateTrie.Update", "err", err) } } @@ -121,8 +143,8 @@ func (t *SecureTrie) Update(key, value []byte) { // The value bytes must not be modified by the caller while they are // stored in the trie. // -// If a node was not found in the database, a MissingNodeError is returned. -func (t *SecureTrie) TryUpdate(key, value []byte) error { +// If a node is not found in the database, a MissingNodeError is returned. +func (t *StateTrie) TryUpdate(key, value []byte) error { hk := t.hashKey(key) err := t.trie.TryUpdate(hk, value) if err != nil { @@ -132,16 +154,39 @@ func (t *SecureTrie) TryUpdate(key, value []byte) error { return nil } +// TryUpdateAccount account will abstract the write of an account to the +// secure trie. +func (t *StateTrie) TryUpdateAccount(key []byte, acc *types.StateAccount) error { + hk := t.hashKey(key) + data, err := rlp.EncodeToBytes(acc) + if err != nil { + return err + } + if err := t.trie.TryUpdate(hk, data); err != nil { + return err + } + t.getSecKeyCache()[string(hk)] = common.CopyBytes(key) + return nil +} + // Delete removes any existing value for key from the trie. -func (t *SecureTrie) Delete(key []byte) { +func (t *StateTrie) Delete(key []byte) { if err := t.TryDelete(key); err != nil { - log.Error(fmt.Sprintf("Unhandled trie error: %v", err)) + log.Error("Unhandled trie error in StateTrie.Delete", "err", err) } } // TryDelete removes any existing value for key from the trie. -// If a node was not found in the database, a MissingNodeError is returned. -func (t *SecureTrie) TryDelete(key []byte) error { +// If the specified trie node is not in the trie, nothing will be changed. +// If a node is not found in the database, a MissingNodeError is returned. +func (t *StateTrie) TryDelete(key []byte) error { + hk := t.hashKey(key) + delete(t.getSecKeyCache(), string(hk)) + return t.trie.TryDelete(hk) +} + +// TryDeleteAccount abstracts an account deletion from the trie. +func (t *StateTrie) TryDeleteAccount(key []byte) error { hk := t.hashKey(key) delete(t.getSecKeyCache(), string(hk)) return t.trie.TryDelete(hk) @@ -149,58 +194,64 @@ func (t *SecureTrie) TryDelete(key []byte) error { // GetKey returns the sha3 preimage of a hashed key that was // previously used to store a value. -func (t *SecureTrie) GetKey(shaKey []byte) []byte { +func (t *StateTrie) GetKey(shaKey []byte) []byte { if key, ok := t.getSecKeyCache()[string(shaKey)]; ok { return key } - return t.trie.db.preimage(common.BytesToHash(shaKey)) + if t.preimages == nil { + return nil + } + return t.preimages.preimage(common.BytesToHash(shaKey)) } -// Commit writes all nodes and the secure hash pre-images to the trie's database. -// Nodes are stored with their sha3 hash as the key. -// -// Committing flushes nodes from memory. Subsequent Get calls will load nodes -// from the database. -func (t *SecureTrie) Commit(onleaf LeafCallback) (common.Hash, int, error) { +// Commit collects all dirty nodes in the trie and replaces them with the +// corresponding node hash. All collected nodes (including dirty leaves if +// collectLeaf is true) will be encapsulated into a nodeset for return. +// The returned nodeset can be nil if the trie is clean (nothing to commit). +// All cached preimages will be also flushed if preimages recording is enabled. +// Once the trie is committed, it's not usable anymore. A new trie must +// be created with new root and updated trie database for following usage +func (t *StateTrie) Commit(collectLeaf bool) (common.Hash, *NodeSet, error) { // Write all the pre-images to the actual disk database if len(t.getSecKeyCache()) > 0 { - if t.trie.db.preimages != nil { // Ugly direct check but avoids the below write lock - t.trie.db.lock.Lock() + if t.preimages != nil { + preimages := make(map[common.Hash][]byte) for hk, key := range t.secKeyCache { - t.trie.db.insertPreimage(common.BytesToHash([]byte(hk)), key) + preimages[common.BytesToHash([]byte(hk))] = key } - t.trie.db.lock.Unlock() + t.preimages.insertPreimage(preimages) } t.secKeyCache = make(map[string][]byte) } - // Commit the trie to its intermediate node database - return t.trie.Commit(onleaf) + // Commit the trie and return its modified nodeset. + return t.trie.Commit(collectLeaf) } -// Hash returns the root hash of SecureTrie. It does not write to the +// Hash returns the root hash of StateTrie. It does not write to the // database and can be used even if the trie doesn't have one. -func (t *SecureTrie) Hash() common.Hash { +func (t *StateTrie) Hash() common.Hash { return t.trie.Hash() } -// Copy returns a copy of SecureTrie. -func (t *SecureTrie) Copy() *SecureTrie { - return &SecureTrie{ +// Copy returns a copy of StateTrie. +func (t *StateTrie) Copy() *StateTrie { + return &StateTrie{ trie: *t.trie.Copy(), + preimages: t.preimages, secKeyCache: t.secKeyCache, } } // NodeIterator returns an iterator that returns nodes of the underlying trie. Iteration // starts at the key after the given start key. -func (t *SecureTrie) NodeIterator(start []byte) NodeIterator { +func (t *StateTrie) NodeIterator(start []byte) NodeIterator { return t.trie.NodeIterator(start) } // hashKey returns the hash of key as an ephemeral buffer. // The caller must not hold onto the return value because it will become // invalid on the next call to hashKey or secKey. -func (t *SecureTrie) hashKey(key []byte) []byte { +func (t *StateTrie) hashKey(key []byte) []byte { h := newHasher(false) h.sha.Reset() h.sha.Write(key) @@ -212,7 +263,7 @@ func (t *SecureTrie) hashKey(key []byte) []byte { // getSecKeyCache returns the current secure key cache, creating a new one if // ownership changed (i.e. the current secure trie is a copy of another owning // the actual cache). -func (t *SecureTrie) getSecKeyCache() map[string][]byte { +func (t *StateTrie) getSecKeyCache() map[string][]byte { if t != t.secKeyCacheOwner { t.secKeyCacheOwner = t t.secKeyCache = make(map[string][]byte) diff --git a/trie/secure_trie_test.go b/trie/secure_trie_test.go index a3ece84b57120..ab8462607d998 100644 --- a/trie/secure_trie_test.go +++ b/trie/secure_trie_test.go @@ -18,6 +18,7 @@ package trie import ( "bytes" + "fmt" "runtime" "sync" "testing" @@ -27,16 +28,16 @@ import ( "github.com/ethereum/go-ethereum/ethdb/memorydb" ) -func newEmptySecure() *SecureTrie { - trie, _ := NewSecure(common.Hash{}, NewDatabase(memorydb.New())) +func newEmptySecure() *StateTrie { + trie, _ := NewStateTrie(TrieID(common.Hash{}), NewDatabase(memorydb.New())) return trie } -// makeTestSecureTrie creates a large enough secure trie for testing. -func makeTestSecureTrie() (*Database, *SecureTrie, map[string][]byte) { +// makeTestStateTrie creates a large enough secure trie for testing. +func makeTestStateTrie() (*Database, *StateTrie, map[string][]byte) { // Create an empty trie triedb := NewDatabase(memorydb.New()) - trie, _ := NewSecure(common.Hash{}, triedb) + trie, _ := NewStateTrie(TrieID(common.Hash{}), triedb) // Fill it with some arbitrary data content := make(map[string][]byte) @@ -57,9 +58,15 @@ func makeTestSecureTrie() (*Database, *SecureTrie, map[string][]byte) { trie.Update(key, val) } } - trie.Commit(nil) - - // Return the generated trie + root, nodes, err := trie.Commit(false) + if err != nil { + panic(fmt.Errorf("failed to commit trie %v", err)) + } + if err := triedb.Update(NewWithNodeSet(nodes)); err != nil { + panic(fmt.Errorf("failed to commit db %v", err)) + } + // Re-create the trie based on the new state + trie, _ = NewStateTrie(TrieID(root), triedb) return triedb, trie, content } @@ -105,16 +112,16 @@ func TestSecureGetKey(t *testing.T) { } } -func TestSecureTrieConcurrency(t *testing.T) { +func TestStateTrieConcurrency(t *testing.T) { // Create an initial trie and copy if for concurrent access - _, trie, _ := makeTestSecureTrie() + _, trie, _ := makeTestStateTrie() threads := runtime.NumCPU() - tries := make([]*SecureTrie, threads) + tries := make([]*StateTrie, threads) for i := 0; i < threads; i++ { tries[i] = trie.Copy() } - // Start a batch of goroutines interactng with the trie + // Start a batch of goroutines interacting with the trie pend := new(sync.WaitGroup) pend.Add(threads) for i := 0; i < threads; i++ { @@ -135,7 +142,7 @@ func TestSecureTrieConcurrency(t *testing.T) { tries[index].Update(key, val) } } - tries[index].Commit(nil) + tries[index].Commit(false) }(i) } // Wait for all threads to finish diff --git a/trie/stacktrie.go b/trie/stacktrie.go index b38bb01b0fb32..d37375d35d52e 100644 --- a/trie/stacktrie.go +++ b/trie/stacktrie.go @@ -21,7 +21,6 @@ import ( "bytes" "encoding/gob" "errors" - "fmt" "io" "sync" @@ -38,9 +37,10 @@ var stPool = sync.Pool{ }, } -func stackTrieFromPool(db ethdb.KeyValueWriter) *StackTrie { +func stackTrieFromPool(db ethdb.KeyValueWriter, owner common.Hash) *StackTrie { st := stPool.Get().(*StackTrie) st.db = db + st.owner = owner return st } @@ -53,6 +53,7 @@ func returnToPool(st *StackTrie) { // in order. Once it determines that a subtree will no longer be inserted // into, it will hash it and free up the memory it uses. type StackTrie struct { + owner common.Hash // the owner of the trie nodeType uint8 // node type (as in branch, ext, leaf) val []byte // value contained by this node if it's a leaf key []byte // key chunk covered by this (leaf|ext) node @@ -68,6 +69,16 @@ func NewStackTrie(db ethdb.KeyValueWriter) *StackTrie { } } +// NewStackTrieWithOwner allocates and initializes an empty trie, but with +// the additional owner field. +func NewStackTrieWithOwner(db ethdb.KeyValueWriter, owner common.Hash) *StackTrie { + return &StackTrie{ + owner: owner, + nodeType: emptyNode, + db: db, + } +} + // NewFromBinary initialises a serialized stacktrie with the given db. func NewFromBinary(data []byte, db ethdb.KeyValueWriter) (*StackTrie, error) { var st StackTrie @@ -88,10 +99,12 @@ func (st *StackTrie) MarshalBinary() (data []byte, err error) { w = bufio.NewWriter(&b) ) if err := gob.NewEncoder(w).Encode(struct { - Nodetype uint8 + Owner common.Hash + NodeType uint8 Val []byte Key []byte }{ + st.owner, st.nodeType, st.val, st.key, @@ -122,12 +135,14 @@ func (st *StackTrie) UnmarshalBinary(data []byte) error { func (st *StackTrie) unmarshalBinary(r io.Reader) error { var dec struct { - Nodetype uint8 + Owner common.Hash + NodeType uint8 Val []byte Key []byte } gob.NewDecoder(r).Decode(&dec) - st.nodeType = dec.Nodetype + st.owner = dec.Owner + st.nodeType = dec.NodeType st.val = dec.Val st.key = dec.Key @@ -154,16 +169,16 @@ func (st *StackTrie) setDb(db ethdb.KeyValueWriter) { } } -func newLeaf(key, val []byte, db ethdb.KeyValueWriter) *StackTrie { - st := stackTrieFromPool(db) +func newLeaf(owner common.Hash, key, val []byte, db ethdb.KeyValueWriter) *StackTrie { + st := stackTrieFromPool(db, owner) st.nodeType = leafNode st.key = append(st.key, key...) st.val = val return st } -func newExt(key []byte, child *StackTrie, db ethdb.KeyValueWriter) *StackTrie { - st := stackTrieFromPool(db) +func newExt(owner common.Hash, key []byte, child *StackTrie, db ethdb.KeyValueWriter) *StackTrie { + st := stackTrieFromPool(db, owner) st.nodeType = extNode st.key = append(st.key, key...) st.children[0] = child @@ -191,11 +206,12 @@ func (st *StackTrie) TryUpdate(key, value []byte) error { func (st *StackTrie) Update(key, value []byte) { if err := st.TryUpdate(key, value); err != nil { - log.Error(fmt.Sprintf("Unhandled trie error: %v", err)) + log.Error("Unhandled trie error in StackTrie.Update", "err", err) } } func (st *StackTrie) Reset() { + st.owner = common.Hash{} st.db = nil st.key = st.key[:0] st.val = nil @@ -236,7 +252,7 @@ func (st *StackTrie) insert(key, value []byte) { // Add new child if st.children[idx] == nil { - st.children[idx] = newLeaf(key[1:], value, st.db) + st.children[idx] = newLeaf(st.owner, key[1:], value, st.db) } else { st.children[idx].insert(key[1:], value) } @@ -262,7 +278,7 @@ func (st *StackTrie) insert(key, value []byte) { // node directly. var n *StackTrie if diffidx < len(st.key)-1 { - n = newExt(st.key[diffidx+1:], st.children[0], st.db) + n = newExt(st.owner, st.key[diffidx+1:], st.children[0], st.db) } else { // Break on the last byte, no need to insert // an extension node: reuse the current node @@ -282,12 +298,12 @@ func (st *StackTrie) insert(key, value []byte) { // the common prefix is at least one byte // long, insert a new intermediate branch // node. - st.children[0] = stackTrieFromPool(st.db) + st.children[0] = stackTrieFromPool(st.db, st.owner) st.children[0].nodeType = branchNode p = st.children[0] } // Create a leaf for the inserted part - o := newLeaf(key[diffidx+1:], value, st.db) + o := newLeaf(st.owner, key[diffidx+1:], value, st.db) // Insert both child leaves where they belong: origIdx := st.key[diffidx] @@ -323,7 +339,7 @@ func (st *StackTrie) insert(key, value []byte) { // Convert current node into an ext, // and insert a child branch node. st.nodeType = extNode - st.children[0] = NewStackTrie(st.db) + st.children[0] = NewStackTrieWithOwner(st.db, st.owner) st.children[0].nodeType = branchNode p = st.children[0] } @@ -332,11 +348,11 @@ func (st *StackTrie) insert(key, value []byte) { // value and another containing the new value. The child leaf // is hashed directly in order to free up some memory. origIdx := st.key[diffidx] - p.children[origIdx] = newLeaf(st.key[diffidx+1:], st.val, st.db) + p.children[origIdx] = newLeaf(st.owner, st.key[diffidx+1:], st.val, st.db) p.children[origIdx].hash() newIdx := key[diffidx] - p.children[newIdx] = newLeaf(key[diffidx+1:], value, st.db) + p.children[newIdx] = newLeaf(st.owner, key[diffidx+1:], value, st.db) // Finally, cut off the key part that has been passed // over to the children. @@ -359,11 +375,12 @@ func (st *StackTrie) insert(key, value []byte) { // hash converts st into a 'hashedNode', if possible. Possible outcomes: // // 1. The rlp-encoded value was >= 32 bytes: -// - Then the 32-byte `hash` will be accessible in `st.val`. -// - And the 'st.type' will be 'hashedNode' +// - Then the 32-byte `hash` will be accessible in `st.val`. +// - And the 'st.type' will be 'hashedNode' +// // 2. The rlp-encoded value was < 32 bytes -// - Then the <32 byte rlp-encoded value will be accessible in 'st.val'. -// - And the 'st.type' will be 'hashedNode' AGAIN +// - Then the <32 byte rlp-encoded value will be accessible in 'st.val'. +// - And the 'st.type' will be 'hashedNode' AGAIN // // This method also sets 'st.type' to hashedNode, and clears 'st.key'. func (st *StackTrie) hash() { diff --git a/trie/stacktrie_test.go b/trie/stacktrie_test.go index e57df60369bf2..069e4981d71ab 100644 --- a/trie/stacktrie_test.go +++ b/trie/stacktrie_test.go @@ -188,7 +188,7 @@ func TestStackTrieInsertAndHash(t *testing.T) { func TestSizeBug(t *testing.T) { st := NewStackTrie(nil) - nt, _ := New(common.Hash{}, NewDatabase(memorydb.New())) + nt := NewEmpty(NewDatabase(memorydb.New())) leaf := common.FromHex("290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563") value := common.FromHex("94cf40d0d2b44f2b66e07cace1372ca42b73cf21a3") @@ -203,7 +203,7 @@ func TestSizeBug(t *testing.T) { func TestEmptyBug(t *testing.T) { st := NewStackTrie(nil) - nt, _ := New(common.Hash{}, NewDatabase(memorydb.New())) + nt := NewEmpty(NewDatabase(memorydb.New())) //leaf := common.FromHex("290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563") //value := common.FromHex("94cf40d0d2b44f2b66e07cace1372ca42b73cf21a3") @@ -229,7 +229,7 @@ func TestEmptyBug(t *testing.T) { func TestValLength56(t *testing.T) { st := NewStackTrie(nil) - nt, _ := New(common.Hash{}, NewDatabase(memorydb.New())) + nt := NewEmpty(NewDatabase(memorydb.New())) //leaf := common.FromHex("290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563") //value := common.FromHex("94cf40d0d2b44f2b66e07cace1372ca42b73cf21a3") @@ -254,7 +254,8 @@ func TestValLength56(t *testing.T) { // which causes a lot of node-within-node. This case was found via fuzzing. func TestUpdateSmallNodes(t *testing.T) { st := NewStackTrie(nil) - nt, _ := New(common.Hash{}, NewDatabase(memorydb.New())) + nt := NewEmpty(NewDatabase(memorydb.New())) + kvs := []struct { K string V string @@ -282,7 +283,8 @@ func TestUpdateSmallNodes(t *testing.T) { func TestUpdateVariableKeys(t *testing.T) { t.SkipNow() st := NewStackTrie(nil) - nt, _ := New(common.Hash{}, NewDatabase(memorydb.New())) + nt := NewEmpty(NewDatabase(memorydb.New())) + kvs := []struct { K string V string @@ -343,7 +345,6 @@ func TestStacktrieNotModifyValues(t *testing.T) { if !bytes.Equal(have, want) { t.Fatalf("item %d, have %#x want %#x", i, have, want) } - } } @@ -352,7 +353,7 @@ func TestStacktrieNotModifyValues(t *testing.T) { func TestStacktrieSerialization(t *testing.T) { var ( st = NewStackTrie(nil) - nt, _ = New(common.Hash{}, NewDatabase(memorydb.New())) + nt = NewEmpty(NewDatabase(memorydb.New())) keyB = big.NewInt(1) keyDelta = big.NewInt(1) vals [][]byte diff --git a/trie/sync.go b/trie/sync.go index db51dd4b036af..31d3cbe91b9e1 100644 --- a/trie/sync.go +++ b/trie/sync.go @@ -19,11 +19,13 @@ package trie import ( "errors" "fmt" + "sync" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/prque" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/log" ) // ErrNotRequested is returned by the trie sync when it's requested to process a @@ -39,19 +41,6 @@ var ErrAlreadyProcessed = errors.New("already processed") // memory if the node was configured with a significant number of peers. const maxFetchesPerDepth = 16384 -// request represents a scheduled or already in-flight state retrieval request. -type request struct { - path []byte // Merkle path leading to this node for prioritization - hash common.Hash // Hash of the node data content to retrieve - data []byte // Data content of the node, cached until all subtrees complete - code bool // Whether this is a code entry - - parents []*request // Parent state nodes referencing this entry (notify all upon completion) - deps int // Number of dependencies before allowed to commit this node - - callback LeafCallback // Callback to invoke if a leaf node it reached on this branch -} - // SyncPath is a path tuple identifying a particular trie node either in a single // trie (account) or a layered trie (account -> storage). // @@ -85,30 +74,58 @@ func NewSyncPath(path []byte) SyncPath { return SyncPath{hexToKeybytes(path[:64]), hexToCompact(path[64:])} } -// SyncResult is a response with requested data along with it's hash. -type SyncResult struct { - Hash common.Hash // Hash of the originally unknown trie node - Data []byte // Data content of the retrieved node +// nodeRequest represents a scheduled or already in-flight trie node retrieval request. +type nodeRequest struct { + hash common.Hash // Hash of the trie node to retrieve + path []byte // Merkle path leading to this node for prioritization + data []byte // Data content of the node, cached until all subtrees complete + + parent *nodeRequest // Parent state node referencing this entry + deps int // Number of dependencies before allowed to commit this node + callback LeafCallback // Callback to invoke if a leaf node it reached on this branch +} + +// codeRequest represents a scheduled or already in-flight bytecode retrieval request. +type codeRequest struct { + hash common.Hash // Hash of the contract bytecode to retrieve + path []byte // Merkle path leading to this node for prioritization + data []byte // Data content of the node, cached until all subtrees complete + parents []*nodeRequest // Parent state nodes referencing this entry (notify all upon completion) +} + +// NodeSyncResult is a response with requested trie node along with its node path. +type NodeSyncResult struct { + Path string // Path of the originally unknown trie node + Data []byte // Data content of the retrieved trie node +} + +// CodeSyncResult is a response with requested bytecode along with its hash. +type CodeSyncResult struct { + Hash common.Hash // Hash the originally unknown bytecode + Data []byte // Data content of the retrieved bytecode } // syncMemBatch is an in-memory buffer of successfully downloaded but not yet // persisted data items. type syncMemBatch struct { - nodes map[common.Hash][]byte // In-memory membatch of recently completed nodes - codes map[common.Hash][]byte // In-memory membatch of recently completed codes + nodes map[string][]byte // In-memory membatch of recently completed nodes + hashes map[string]common.Hash // Hashes of recently completed nodes + codes map[common.Hash][]byte // In-memory membatch of recently completed codes + size uint64 // Estimated batch-size of in-memory data. } // newSyncMemBatch allocates a new memory-buffer for not-yet persisted trie nodes. func newSyncMemBatch() *syncMemBatch { return &syncMemBatch{ - nodes: make(map[common.Hash][]byte), - codes: make(map[common.Hash][]byte), + nodes: make(map[string][]byte), + hashes: make(map[string]common.Hash), + codes: make(map[common.Hash][]byte), } } -// hasNode reports the trie node with specific hash is already cached. -func (batch *syncMemBatch) hasNode(hash common.Hash) bool { - _, ok := batch.nodes[hash] +// hasNode reports the trie node with specific path is already cached. +func (batch *syncMemBatch) hasNode(path []byte) bool { + _, ok := batch.nodes[string(path)] return ok } @@ -122,12 +139,12 @@ func (batch *syncMemBatch) hasCode(hash common.Hash) bool { // unknown trie hashes to retrieve, accepts node data associated with said hashes // and reconstructs the trie step by step until all is done. type Sync struct { - database ethdb.KeyValueReader // Persistent database to check for existing entries - membatch *syncMemBatch // Memory buffer to avoid frequent database writes - nodeReqs map[common.Hash]*request // Pending requests pertaining to a trie node hash - codeReqs map[common.Hash]*request // Pending requests pertaining to a code hash - queue *prque.Prque // Priority queue with the pending requests - fetches map[int]int // Number of active fetches per trie node depth + database ethdb.KeyValueReader // Persistent database to check for existing entries + membatch *syncMemBatch // Memory buffer to avoid frequent database writes + nodeReqs map[string]*nodeRequest // Pending requests pertaining to a trie node path + codeReqs map[common.Hash]*codeRequest // Pending requests pertaining to a code hash + queue *prque.Prque // Priority queue with the pending requests + fetches map[int]int // Number of active fetches per trie node depth } // NewSync creates a new trie data download scheduler. @@ -135,51 +152,51 @@ func NewSync(root common.Hash, database ethdb.KeyValueReader, callback LeafCallb ts := &Sync{ database: database, membatch: newSyncMemBatch(), - nodeReqs: make(map[common.Hash]*request), - codeReqs: make(map[common.Hash]*request), + nodeReqs: make(map[string]*nodeRequest), + codeReqs: make(map[common.Hash]*codeRequest), queue: prque.New(nil), fetches: make(map[int]int), } - ts.AddSubTrie(root, nil, common.Hash{}, callback) + ts.AddSubTrie(root, nil, common.Hash{}, nil, callback) return ts } -// AddSubTrie registers a new trie to the sync code, rooted at the designated parent. -func (s *Sync) AddSubTrie(root common.Hash, path []byte, parent common.Hash, callback LeafCallback) { +// AddSubTrie registers a new trie to the sync code, rooted at the designated +// parent for completion tracking. The given path is a unique node path in +// hex format and contain all the parent path if it's layered trie node. +func (s *Sync) AddSubTrie(root common.Hash, path []byte, parent common.Hash, parentPath []byte, callback LeafCallback) { // Short circuit if the trie is empty or already known if root == emptyRoot { return } - if s.membatch.hasNode(root) { + if s.membatch.hasNode(path) { return } - // If database says this is a duplicate, then at least the trie node is - // present, and we hold the assumption that it's NOT legacy contract code. if rawdb.HasTrieNode(s.database, root) { return } // Assemble the new sub-trie sync request - req := &request{ - path: path, + req := &nodeRequest{ hash: root, + path: path, callback: callback, } // If this sub-trie has a designated parent, link them together if parent != (common.Hash{}) { - ancestor := s.nodeReqs[parent] + ancestor := s.nodeReqs[string(parentPath)] if ancestor == nil { panic(fmt.Sprintf("sub-trie ancestor not found: %x", parent)) } ancestor.deps++ - req.parents = append(req.parents, ancestor) + req.parent = ancestor } - s.schedule(req) + s.scheduleNodeRequest(req) } // AddCodeEntry schedules the direct retrieval of a contract code that should not // be interpreted as a trie node, but rather accepted and stored into the database // as is. -func (s *Sync) AddCodeEntry(hash common.Hash, path []byte, parent common.Hash) { +func (s *Sync) AddCodeEntry(hash common.Hash, path []byte, parent common.Hash, parentPath []byte) { // Short circuit if the entry is empty or already known if hash == emptyState { return @@ -196,30 +213,29 @@ func (s *Sync) AddCodeEntry(hash common.Hash, path []byte, parent common.Hash) { return } // Assemble the new sub-trie sync request - req := &request{ + req := &codeRequest{ path: path, hash: hash, - code: true, } // If this sub-trie has a designated parent, link them together if parent != (common.Hash{}) { - ancestor := s.nodeReqs[parent] // the parent of codereq can ONLY be nodereq + ancestor := s.nodeReqs[string(parentPath)] // the parent of codereq can ONLY be nodereq if ancestor == nil { panic(fmt.Sprintf("raw-entry ancestor not found: %x", parent)) } ancestor.deps++ req.parents = append(req.parents, ancestor) } - s.schedule(req) + s.scheduleCodeRequest(req) } // Missing retrieves the known missing nodes from the trie for retrieval. To aid // both eth/6x style fast sync and snap/1x style state sync, the paths of trie // nodes are returned too, as well as separate hash list for codes. -func (s *Sync) Missing(max int) (nodes []common.Hash, paths []SyncPath, codes []common.Hash) { +func (s *Sync) Missing(max int) ([]string, []common.Hash, []common.Hash) { var ( + nodePaths []string nodeHashes []common.Hash - nodePaths []SyncPath codeHashes []common.Hash ) for !s.queue.Empty() && (max == 0 || len(nodeHashes)+len(codeHashes) < max) { @@ -235,62 +251,76 @@ func (s *Sync) Missing(max int) (nodes []common.Hash, paths []SyncPath, codes [] s.queue.Pop() s.fetches[depth]++ - hash := item.(common.Hash) - if req, ok := s.nodeReqs[hash]; ok { - nodeHashes = append(nodeHashes, hash) - nodePaths = append(nodePaths, NewSyncPath(req.path)) - } else { - codeHashes = append(codeHashes, hash) + switch item := item.(type) { + case common.Hash: + codeHashes = append(codeHashes, item) + case string: + req, ok := s.nodeReqs[item] + if !ok { + log.Error("Missing node request", "path", item) + continue // System very wrong, shouldn't happen + } + nodePaths = append(nodePaths, item) + nodeHashes = append(nodeHashes, req.hash) } } - return nodeHashes, nodePaths, codeHashes + return nodePaths, nodeHashes, codeHashes } -// Process injects the received data for requested item. Note it can +// ProcessCode injects the received data for requested item. Note it can // happpen that the single response commits two pending requests(e.g. // there are two requests one for code and one for node but the hash // is same). In this case the second response for the same hash will // be treated as "non-requested" item or "already-processed" item but // there is no downside. -func (s *Sync) Process(result SyncResult) error { - // If the item was not requested either for code or node, bail out - if s.nodeReqs[result.Hash] == nil && s.codeReqs[result.Hash] == nil { +func (s *Sync) ProcessCode(result CodeSyncResult) error { + // If the code was not requested or it's already processed, bail out + req := s.codeReqs[result.Hash] + if req == nil { return ErrNotRequested } - // There is an pending code request for this data, commit directly - var filled bool - if req := s.codeReqs[result.Hash]; req != nil && req.data == nil { - filled = true - req.data = result.Data - s.commit(req) - } - // There is an pending node request for this data, fill it. - if req := s.nodeReqs[result.Hash]; req != nil && req.data == nil { - filled = true - // Decode the node data content and update the request - node, err := decodeNode(result.Hash[:], result.Data) - if err != nil { - return err - } - req.data = result.Data + if req.data != nil { + return ErrAlreadyProcessed + } + req.data = result.Data + return s.commitCodeRequest(req) +} - // Create and schedule a request for all the children nodes - requests, err := s.children(req, node) - if err != nil { - return err - } - if len(requests) == 0 && req.deps == 0 { - s.commit(req) - } else { - req.deps += len(requests) - for _, child := range requests { - s.schedule(child) - } - } +// ProcessNode injects the received data for requested item. Note it can +// happen that the single response commits two pending requests(e.g. +// there are two requests one for code and one for node but the hash +// is same). In this case the second response for the same hash will +// be treated as "non-requested" item or "already-processed" item but +// there is no downside. +func (s *Sync) ProcessNode(result NodeSyncResult) error { + // If the trie node was not requested or it's already processed, bail out + req := s.nodeReqs[result.Path] + if req == nil { + return ErrNotRequested } - if !filled { + if req.data != nil { return ErrAlreadyProcessed } + // Decode the node data content and update the request + node, err := decodeNode(req.hash.Bytes(), result.Data) + if err != nil { + return err + } + req.data = result.Data + + // Create and schedule a request for all the children nodes + requests, err := s.children(req, node) + if err != nil { + return err + } + if len(requests) == 0 && req.deps == 0 { + s.commitNodeRequest(req) + } else { + req.deps += len(requests) + for _, child := range requests { + s.scheduleNodeRequest(child) + } + } return nil } @@ -298,17 +328,22 @@ func (s *Sync) Process(result SyncResult) error { // storage, returning any occurred error. func (s *Sync) Commit(dbw ethdb.Batch) error { // Dump the membatch into a database dbw - for key, value := range s.membatch.nodes { - rawdb.WriteTrieNode(dbw, key, value) + for path, value := range s.membatch.nodes { + rawdb.WriteTrieNode(dbw, s.membatch.hashes[path], value) } - for key, value := range s.membatch.codes { - rawdb.WriteCode(dbw, key, value) + for hash, value := range s.membatch.codes { + rawdb.WriteCode(dbw, hash, value) } // Drop the membatch data and return s.membatch = newSyncMemBatch() return nil } +// MemSize returns an estimated size (in bytes) of the data held in the membatch. +func (s *Sync) MemSize() uint64 { + return s.membatch.size +} + // Pending returns the number of state entries currently pending for download. func (s *Sync) Pending() int { return len(s.nodeReqs) + len(s.codeReqs) @@ -317,23 +352,31 @@ func (s *Sync) Pending() int { // schedule inserts a new state retrieval request into the fetch queue. If there // is already a pending request for this node, the new request will be discarded // and only a parent reference added to the old one. -func (s *Sync) schedule(req *request) { - var reqset = s.nodeReqs - if req.code { - reqset = s.codeReqs +func (s *Sync) scheduleNodeRequest(req *nodeRequest) { + s.nodeReqs[string(req.path)] = req + + // Schedule the request for future retrieval. This queue is shared + // by both node requests and code requests. + prio := int64(len(req.path)) << 56 // depth >= 128 will never happen, storage leaves will be included in their parents + for i := 0; i < 14 && i < len(req.path); i++ { + prio |= int64(15-req.path[i]) << (52 - i*4) // 15-nibble => lexicographic order } + s.queue.Push(string(req.path), prio) +} + +// schedule inserts a new state retrieval request into the fetch queue. If there +// is already a pending request for this node, the new request will be discarded +// and only a parent reference added to the old one. +func (s *Sync) scheduleCodeRequest(req *codeRequest) { // If we're already requesting this node, add a new reference and stop - if old, ok := reqset[req.hash]; ok { + if old, ok := s.codeReqs[req.hash]; ok { old.parents = append(old.parents, req.parents...) return } - reqset[req.hash] = req + s.codeReqs[req.hash] = req // Schedule the request for future retrieval. This queue is shared - // by both node requests and code requests. It can happen that there - // is a trie node and code has same hash. In this case two elements - // with same hash and same or different depth will be pushed. But it's - // ok the worst case is the second response will be treated as duplicated. + // by both node requests and code requests. prio := int64(len(req.path)) << 56 // depth >= 128 will never happen, storage leaves will be included in their parents for i := 0; i < 14 && i < len(req.path); i++ { prio |= int64(15-req.path[i]) << (52 - i*4) // 15-nibble => lexicographic order @@ -343,13 +386,13 @@ func (s *Sync) schedule(req *request) { // children retrieves all the missing children of a state trie entry for future // retrieval scheduling. -func (s *Sync) children(req *request, object node) ([]*request, error) { +func (s *Sync) children(req *nodeRequest, object node) ([]*nodeRequest, error) { // Gather all the children of the node, irrelevant whether known or not - type child struct { + type childNode struct { path []byte node node } - var children []child + var children []childNode switch node := (object).(type) { case *shortNode: @@ -357,14 +400,14 @@ func (s *Sync) children(req *request, object node) ([]*request, error) { if hasTerm(key) { key = key[:len(key)-1] } - children = []child{{ + children = []childNode{{ node: node.Val, path: append(append([]byte(nil), req.path...), key...), }} case *fullNode: for i := 0; i < 17; i++ { if node.Children[i] != nil { - children = append(children, child{ + children = append(children, childNode{ node: node.Children[i], path: append(append([]byte(nil), req.path...), byte(i)), }) @@ -374,7 +417,10 @@ func (s *Sync) children(req *request, object node) ([]*request, error) { panic(fmt.Sprintf("unknown node: %+v", node)) } // Iterate over the children, and request all unknown ones - requests := make([]*request, 0, len(children)) + var ( + missing = make(chan *nodeRequest, len(children)) + pending sync.WaitGroup + ) for _, child := range children { // Notify any external watcher of a new key/value node if req.callback != nil { @@ -386,7 +432,7 @@ func (s *Sync) children(req *request, object node) ([]*request, error) { paths = append(paths, hexToKeybytes(child.path[:2*common.HashLength])) paths = append(paths, hexToKeybytes(child.path[2*common.HashLength:])) } - if err := req.callback(paths, child.path, node, req.hash); err != nil { + if err := req.callback(paths, child.path, node, req.hash, req.path); err != nil { return nil, err } } @@ -394,22 +440,39 @@ func (s *Sync) children(req *request, object node) ([]*request, error) { // If the child references another node, resolve or schedule if node, ok := (child.node).(hashNode); ok { // Try to resolve the node from the local database - hash := common.BytesToHash(node) - if s.membatch.hasNode(hash) { + if s.membatch.hasNode(child.path) { continue } - // If database says duplicate, then at least the trie node is present - // and we hold the assumption that it's NOT legacy contract code. - if rawdb.HasTrieNode(s.database, hash) { - continue - } - // Locally unknown node, schedule for retrieval - requests = append(requests, &request{ - path: child.path, - hash: hash, - parents: []*request{req}, - callback: req.callback, - }) + // Check the presence of children concurrently + pending.Add(1) + go func(child childNode) { + defer pending.Done() + + // If database says duplicate, then at least the trie node is present + // and we hold the assumption that it's NOT legacy contract code. + chash := common.BytesToHash(node) + if rawdb.HasTrieNode(s.database, chash) { + return + } + // Locally unknown node, schedule for retrieval + missing <- &nodeRequest{ + path: child.path, + hash: chash, + parent: req, + callback: req.callback, + } + }(child) + } + } + pending.Wait() + + requests := make([]*nodeRequest, 0, len(children)) + for done := false; !done; { + select { + case miss := <-missing: + requests = append(requests, miss) + default: + done = true } } return requests, nil @@ -418,22 +481,44 @@ func (s *Sync) children(req *request, object node) ([]*request, error) { // commit finalizes a retrieval request and stores it into the membatch. If any // of the referencing parent requests complete due to this commit, they are also // committed themselves. -func (s *Sync) commit(req *request) (err error) { +func (s *Sync) commitNodeRequest(req *nodeRequest) error { // Write the node content to the membatch - if req.code { - s.membatch.codes[req.hash] = req.data - delete(s.codeReqs, req.hash) - s.fetches[len(req.path)]-- - } else { - s.membatch.nodes[req.hash] = req.data - delete(s.nodeReqs, req.hash) - s.fetches[len(req.path)]-- + s.membatch.nodes[string(req.path)] = req.data + s.membatch.hashes[string(req.path)] = req.hash + // The size tracking refers to the db-batch, not the in-memory data. + // Therefore, we ignore the req.path, and account only for the hash+data + // which eventually is written to db. + s.membatch.size += common.HashLength + uint64(len(req.data)) + delete(s.nodeReqs, string(req.path)) + s.fetches[len(req.path)]-- + + // Check parent for completion + if req.parent != nil { + req.parent.deps-- + if req.parent.deps == 0 { + if err := s.commitNodeRequest(req.parent); err != nil { + return err + } + } } + return nil +} + +// commit finalizes a retrieval request and stores it into the membatch. If any +// of the referencing parent requests complete due to this commit, they are also +// committed themselves. +func (s *Sync) commitCodeRequest(req *codeRequest) error { + // Write the node content to the membatch + s.membatch.codes[req.hash] = req.data + s.membatch.size += common.HashLength + uint64(len(req.data)) + delete(s.codeReqs, req.hash) + s.fetches[len(req.path)]-- + // Check all parents for completion for _, parent := range req.parents { parent.deps-- if parent.deps == 0 { - if err := s.commit(parent); err != nil { + if err := s.commitNodeRequest(parent); err != nil { return err } } diff --git a/trie/sync_test.go b/trie/sync_test.go index 970730b671870..a02527855300c 100644 --- a/trie/sync_test.go +++ b/trie/sync_test.go @@ -18,6 +18,7 @@ package trie import ( "bytes" + "fmt" "testing" "github.com/ethereum/go-ethereum/common" @@ -26,10 +27,10 @@ import ( ) // makeTestTrie create a sample test trie to test node-wise reconstruction. -func makeTestTrie() (*Database, *SecureTrie, map[string][]byte) { +func makeTestTrie() (*Database, *StateTrie, map[string][]byte) { // Create an empty trie triedb := NewDatabase(memorydb.New()) - trie, _ := NewSecure(common.Hash{}, triedb) + trie, _ := NewStateTrie(TrieID(common.Hash{}), triedb) // Fill it with some arbitrary data content := make(map[string][]byte) @@ -50,9 +51,15 @@ func makeTestTrie() (*Database, *SecureTrie, map[string][]byte) { trie.Update(key, val) } } - trie.Commit(nil) - - // Return the generated trie + root, nodes, err := trie.Commit(false) + if err != nil { + panic(fmt.Errorf("failed to commit trie %v", err)) + } + if err := triedb.Update(NewWithNodeSet(nodes)); err != nil { + panic(fmt.Errorf("failed to commit db %v", err)) + } + // Re-create the trie based on the new state + trie, _ = NewStateTrie(TrieID(root), triedb) return triedb, trie, content } @@ -60,7 +67,7 @@ func makeTestTrie() (*Database, *SecureTrie, map[string][]byte) { // content map. func checkTrieContents(t *testing.T, db *Database, root []byte, content map[string][]byte) { // Check root availability and trie contents - trie, err := NewSecure(common.BytesToHash(root), db) + trie, err := NewStateTrie(TrieID(common.BytesToHash(root)), db) if err != nil { t.Fatalf("failed to create trie at %x: %v", root, err) } @@ -77,7 +84,7 @@ func checkTrieContents(t *testing.T, db *Database, root []byte, content map[stri // checkTrieConsistency checks that all nodes in a trie are indeed present. func checkTrieConsistency(db *Database, root common.Hash) error { // Create and iterate a trie rooted in a subnode - trie, err := NewSecure(root, db) + trie, err := NewStateTrie(TrieID(root), db) if err != nil { return nil // Consider a non existent state consistent } @@ -87,17 +94,24 @@ func checkTrieConsistency(db *Database, root common.Hash) error { return it.Error() } +// trieElement represents the element in the state trie(bytecode or trie node). +type trieElement struct { + path string + hash common.Hash + syncPath SyncPath +} + // Tests that an empty trie is not scheduled for syncing. func TestEmptySync(t *testing.T) { dbA := NewDatabase(memorydb.New()) dbB := NewDatabase(memorydb.New()) - emptyA, _ := New(common.Hash{}, dbA) - emptyB, _ := New(emptyRoot, dbB) + emptyA, _ := New(TrieID(common.Hash{}), dbA) + emptyB, _ := New(TrieID(emptyRoot), dbB) for i, trie := range []*Trie{emptyA, emptyB} { sync := NewSync(trie.Hash(), memorydb.New(), nil) - if nodes, paths, codes := sync.Missing(1); len(nodes) != 0 || len(paths) != 0 || len(codes) != 0 { - t.Errorf("test %d: content requested for empty trie: %v, %v, %v", i, nodes, paths, codes) + if paths, nodes, codes := sync.Missing(1); len(paths) != 0 || len(nodes) != 0 || len(codes) != 0 { + t.Errorf("test %d: content requested for empty trie: %v, %v, %v", i, paths, nodes, codes) } } } @@ -118,35 +132,38 @@ func testIterativeSync(t *testing.T, count int, bypath bool) { triedb := NewDatabase(diskdb) sched := NewSync(srcTrie.Hash(), diskdb, nil) - nodes, paths, codes := sched.Missing(count) - var ( - hashQueue []common.Hash - pathQueue []SyncPath - ) - if !bypath { - hashQueue = append(append(hashQueue[:0], nodes...), codes...) - } else { - hashQueue = append(hashQueue[:0], codes...) - pathQueue = append(pathQueue[:0], paths...) + // The code requests are ignored here since there is no code + // at the testing trie. + paths, nodes, _ := sched.Missing(count) + var elements []trieElement + for i := 0; i < len(paths); i++ { + elements = append(elements, trieElement{ + path: paths[i], + hash: nodes[i], + syncPath: NewSyncPath([]byte(paths[i])), + }) } - for len(hashQueue)+len(pathQueue) > 0 { - results := make([]SyncResult, len(hashQueue)+len(pathQueue)) - for i, hash := range hashQueue { - data, err := srcDb.Node(hash) - if err != nil { - t.Fatalf("failed to retrieve node data for hash %x: %v", hash, err) + for len(elements) > 0 { + results := make([]NodeSyncResult, len(elements)) + if !bypath { + for i, element := range elements { + data, err := srcDb.Node(element.hash) + if err != nil { + t.Fatalf("failed to retrieve node data for hash %x: %v", element.hash, err) + } + results[i] = NodeSyncResult{element.path, data} } - results[i] = SyncResult{hash, data} - } - for i, path := range pathQueue { - data, _, err := srcTrie.TryGetNode(path[0]) - if err != nil { - t.Fatalf("failed to retrieve node data for path %x: %v", path, err) + } else { + for i, element := range elements { + data, _, err := srcTrie.TryGetNode(element.syncPath[len(element.syncPath)-1]) + if err != nil { + t.Fatalf("failed to retrieve node data for path %x: %v", element.path, err) + } + results[i] = NodeSyncResult{element.path, data} } - results[len(hashQueue)+i] = SyncResult{crypto.Keccak256Hash(data), data} } for _, result := range results { - if err := sched.Process(result); err != nil { + if err := sched.ProcessNode(result); err != nil { t.Fatalf("failed to process result %v", err) } } @@ -156,12 +173,14 @@ func testIterativeSync(t *testing.T, count int, bypath bool) { } batch.Write() - nodes, paths, codes = sched.Missing(count) - if !bypath { - hashQueue = append(append(hashQueue[:0], nodes...), codes...) - } else { - hashQueue = append(hashQueue[:0], codes...) - pathQueue = append(pathQueue[:0], paths...) + paths, nodes, _ = sched.Missing(count) + elements = elements[:0] + for i := 0; i < len(paths); i++ { + elements = append(elements, trieElement{ + path: paths[i], + hash: nodes[i], + syncPath: NewSyncPath([]byte(paths[i])), + }) } } // Cross check that the two tries are in sync @@ -179,21 +198,29 @@ func TestIterativeDelayedSync(t *testing.T) { triedb := NewDatabase(diskdb) sched := NewSync(srcTrie.Hash(), diskdb, nil) - nodes, _, codes := sched.Missing(10000) - queue := append(append([]common.Hash{}, nodes...), codes...) - - for len(queue) > 0 { + // The code requests are ignored here since there is no code + // at the testing trie. + paths, nodes, _ := sched.Missing(10000) + var elements []trieElement + for i := 0; i < len(paths); i++ { + elements = append(elements, trieElement{ + path: paths[i], + hash: nodes[i], + syncPath: NewSyncPath([]byte(paths[i])), + }) + } + for len(elements) > 0 { // Sync only half of the scheduled nodes - results := make([]SyncResult, len(queue)/2+1) - for i, hash := range queue[:len(results)] { - data, err := srcDb.Node(hash) + results := make([]NodeSyncResult, len(elements)/2+1) + for i, element := range elements[:len(results)] { + data, err := srcDb.Node(element.hash) if err != nil { - t.Fatalf("failed to retrieve node data for %x: %v", hash, err) + t.Fatalf("failed to retrieve node data for %x: %v", element.hash, err) } - results[i] = SyncResult{hash, data} + results[i] = NodeSyncResult{element.path, data} } for _, result := range results { - if err := sched.Process(result); err != nil { + if err := sched.ProcessNode(result); err != nil { t.Fatalf("failed to process result %v", err) } } @@ -203,8 +230,15 @@ func TestIterativeDelayedSync(t *testing.T) { } batch.Write() - nodes, _, codes = sched.Missing(10000) - queue = append(append(queue[len(results):], nodes...), codes...) + paths, nodes, _ = sched.Missing(10000) + elements = elements[len(results):] + for i := 0; i < len(paths); i++ { + elements = append(elements, trieElement{ + path: paths[i], + hash: nodes[i], + syncPath: NewSyncPath([]byte(paths[i])), + }) + } } // Cross check that the two tries are in sync checkTrieContents(t, triedb, srcTrie.Hash().Bytes(), srcData) @@ -225,24 +259,30 @@ func testIterativeRandomSync(t *testing.T, count int) { triedb := NewDatabase(diskdb) sched := NewSync(srcTrie.Hash(), diskdb, nil) - queue := make(map[common.Hash]struct{}) - nodes, _, codes := sched.Missing(count) - for _, hash := range append(nodes, codes...) { - queue[hash] = struct{}{} + // The code requests are ignored here since there is no code + // at the testing trie. + paths, nodes, _ := sched.Missing(count) + queue := make(map[string]trieElement) + for i, path := range paths { + queue[path] = trieElement{ + path: paths[i], + hash: nodes[i], + syncPath: NewSyncPath([]byte(paths[i])), + } } for len(queue) > 0 { // Fetch all the queued nodes in a random order - results := make([]SyncResult, 0, len(queue)) - for hash := range queue { - data, err := srcDb.Node(hash) + results := make([]NodeSyncResult, 0, len(queue)) + for path, element := range queue { + data, err := srcDb.Node(element.hash) if err != nil { - t.Fatalf("failed to retrieve node data for %x: %v", hash, err) + t.Fatalf("failed to retrieve node data for %x: %v", element.hash, err) } - results = append(results, SyncResult{hash, data}) + results = append(results, NodeSyncResult{path, data}) } // Feed the retrieved results back and queue new tasks for _, result := range results { - if err := sched.Process(result); err != nil { + if err := sched.ProcessNode(result); err != nil { t.Fatalf("failed to process result %v", err) } } @@ -252,10 +292,14 @@ func testIterativeRandomSync(t *testing.T, count int) { } batch.Write() - queue = make(map[common.Hash]struct{}) - nodes, _, codes = sched.Missing(count) - for _, hash := range append(nodes, codes...) { - queue[hash] = struct{}{} + paths, nodes, _ = sched.Missing(count) + queue = make(map[string]trieElement) + for i, path := range paths { + queue[path] = trieElement{ + path: path, + hash: nodes[i], + syncPath: NewSyncPath([]byte(path)), + } } } // Cross check that the two tries are in sync @@ -273,20 +317,26 @@ func TestIterativeRandomDelayedSync(t *testing.T) { triedb := NewDatabase(diskdb) sched := NewSync(srcTrie.Hash(), diskdb, nil) - queue := make(map[common.Hash]struct{}) - nodes, _, codes := sched.Missing(10000) - for _, hash := range append(nodes, codes...) { - queue[hash] = struct{}{} + // The code requests are ignored here since there is no code + // at the testing trie. + paths, nodes, _ := sched.Missing(10000) + queue := make(map[string]trieElement) + for i, path := range paths { + queue[path] = trieElement{ + path: path, + hash: nodes[i], + syncPath: NewSyncPath([]byte(path)), + } } for len(queue) > 0 { // Sync only half of the scheduled nodes, even those in random order - results := make([]SyncResult, 0, len(queue)/2+1) - for hash := range queue { - data, err := srcDb.Node(hash) + results := make([]NodeSyncResult, 0, len(queue)/2+1) + for path, element := range queue { + data, err := srcDb.Node(element.hash) if err != nil { - t.Fatalf("failed to retrieve node data for %x: %v", hash, err) + t.Fatalf("failed to retrieve node data for %x: %v", element.hash, err) } - results = append(results, SyncResult{hash, data}) + results = append(results, NodeSyncResult{path, data}) if len(results) >= cap(results) { break @@ -294,7 +344,7 @@ func TestIterativeRandomDelayedSync(t *testing.T) { } // Feed the retrieved results back and queue new tasks for _, result := range results { - if err := sched.Process(result); err != nil { + if err := sched.ProcessNode(result); err != nil { t.Fatalf("failed to process result %v", err) } } @@ -304,11 +354,15 @@ func TestIterativeRandomDelayedSync(t *testing.T) { } batch.Write() for _, result := range results { - delete(queue, result.Hash) - } - nodes, _, codes = sched.Missing(10000) - for _, hash := range append(nodes, codes...) { - queue[hash] = struct{}{} + delete(queue, result.Path) + } + paths, nodes, _ = sched.Missing(10000) + for i, path := range paths { + queue[path] = trieElement{ + path: path, + hash: nodes[i], + syncPath: NewSyncPath([]byte(path)), + } } } // Cross check that the two tries are in sync @@ -326,26 +380,35 @@ func TestDuplicateAvoidanceSync(t *testing.T) { triedb := NewDatabase(diskdb) sched := NewSync(srcTrie.Hash(), diskdb, nil) - nodes, _, codes := sched.Missing(0) - queue := append(append([]common.Hash{}, nodes...), codes...) + // The code requests are ignored here since there is no code + // at the testing trie. + paths, nodes, _ := sched.Missing(0) + var elements []trieElement + for i := 0; i < len(paths); i++ { + elements = append(elements, trieElement{ + path: paths[i], + hash: nodes[i], + syncPath: NewSyncPath([]byte(paths[i])), + }) + } requested := make(map[common.Hash]struct{}) - for len(queue) > 0 { - results := make([]SyncResult, len(queue)) - for i, hash := range queue { - data, err := srcDb.Node(hash) + for len(elements) > 0 { + results := make([]NodeSyncResult, len(elements)) + for i, element := range elements { + data, err := srcDb.Node(element.hash) if err != nil { - t.Fatalf("failed to retrieve node data for %x: %v", hash, err) + t.Fatalf("failed to retrieve node data for %x: %v", element.hash, err) } - if _, ok := requested[hash]; ok { - t.Errorf("hash %x already requested once", hash) + if _, ok := requested[element.hash]; ok { + t.Errorf("hash %x already requested once", element.hash) } - requested[hash] = struct{}{} + requested[element.hash] = struct{}{} - results[i] = SyncResult{hash, data} + results[i] = NodeSyncResult{element.path, data} } for _, result := range results { - if err := sched.Process(result); err != nil { + if err := sched.ProcessNode(result); err != nil { t.Fatalf("failed to process result %v", err) } } @@ -355,8 +418,15 @@ func TestDuplicateAvoidanceSync(t *testing.T) { } batch.Write() - nodes, _, codes = sched.Missing(0) - queue = append(append(queue[:0], nodes...), codes...) + paths, nodes, _ = sched.Missing(0) + elements = elements[:0] + for i := 0; i < len(paths); i++ { + elements = append(elements, trieElement{ + path: paths[i], + hash: nodes[i], + syncPath: NewSyncPath([]byte(paths[i])), + }) + } } // Cross check that the two tries are in sync checkTrieContents(t, triedb, srcTrie.Hash().Bytes(), srcData) @@ -373,23 +443,34 @@ func TestIncompleteSync(t *testing.T) { triedb := NewDatabase(diskdb) sched := NewSync(srcTrie.Hash(), diskdb, nil) - var added []common.Hash - - nodes, _, codes := sched.Missing(1) - queue := append(append([]common.Hash{}, nodes...), codes...) - for len(queue) > 0 { + // The code requests are ignored here since there is no code + // at the testing trie. + var ( + added []common.Hash + elements []trieElement + root = srcTrie.Hash() + ) + paths, nodes, _ := sched.Missing(1) + for i := 0; i < len(paths); i++ { + elements = append(elements, trieElement{ + path: paths[i], + hash: nodes[i], + syncPath: NewSyncPath([]byte(paths[i])), + }) + } + for len(elements) > 0 { // Fetch a batch of trie nodes - results := make([]SyncResult, len(queue)) - for i, hash := range queue { - data, err := srcDb.Node(hash) + results := make([]NodeSyncResult, len(elements)) + for i, element := range elements { + data, err := srcDb.Node(element.hash) if err != nil { - t.Fatalf("failed to retrieve node data for %x: %v", hash, err) + t.Fatalf("failed to retrieve node data for %x: %v", element.hash, err) } - results[i] = SyncResult{hash, data} + results[i] = NodeSyncResult{element.path, data} } // Process each of the trie nodes for _, result := range results { - if err := sched.Process(result); err != nil { + if err := sched.ProcessNode(result); err != nil { t.Fatalf("failed to process result %v", err) } } @@ -398,27 +479,36 @@ func TestIncompleteSync(t *testing.T) { t.Fatalf("failed to commit data: %v", err) } batch.Write() + for _, result := range results { - added = append(added, result.Hash) + hash := crypto.Keccak256Hash(result.Data) + if hash != root { + added = append(added, hash) + } // Check that all known sub-tries in the synced trie are complete - if err := checkTrieConsistency(triedb, result.Hash); err != nil { + if err := checkTrieConsistency(triedb, hash); err != nil { t.Fatalf("trie inconsistent: %v", err) } } // Fetch the next batch to retrieve - nodes, _, codes = sched.Missing(1) - queue = append(append(queue[:0], nodes...), codes...) + paths, nodes, _ = sched.Missing(1) + elements = elements[:0] + for i := 0; i < len(paths); i++ { + elements = append(elements, trieElement{ + path: paths[i], + hash: nodes[i], + syncPath: NewSyncPath([]byte(paths[i])), + }) + } } // Sanity check that removing any node from the database is detected - for _, node := range added[1:] { - key := node.Bytes() - value, _ := diskdb.Get(key) - - diskdb.Delete(key) - if err := checkTrieConsistency(triedb, added[0]); err == nil { - t.Fatalf("trie inconsistency not caught, missing: %x", key) + for _, hash := range added { + value, _ := diskdb.Get(hash.Bytes()) + diskdb.Delete(hash.Bytes()) + if err := checkTrieConsistency(triedb, root); err == nil { + t.Fatalf("trie inconsistency not caught, missing: %x", hash) } - diskdb.Put(key, value) + diskdb.Put(hash.Bytes(), value) } } @@ -433,21 +523,33 @@ func TestSyncOrdering(t *testing.T) { triedb := NewDatabase(diskdb) sched := NewSync(srcTrie.Hash(), diskdb, nil) - nodes, paths, _ := sched.Missing(1) - queue := append([]common.Hash{}, nodes...) - reqs := append([]SyncPath{}, paths...) + // The code requests are ignored here since there is no code + // at the testing trie. + var ( + reqs []SyncPath + elements []trieElement + ) + paths, nodes, _ := sched.Missing(1) + for i := 0; i < len(paths); i++ { + elements = append(elements, trieElement{ + path: paths[i], + hash: nodes[i], + syncPath: NewSyncPath([]byte(paths[i])), + }) + reqs = append(reqs, NewSyncPath([]byte(paths[i]))) + } - for len(queue) > 0 { - results := make([]SyncResult, len(queue)) - for i, hash := range queue { - data, err := srcDb.Node(hash) + for len(elements) > 0 { + results := make([]NodeSyncResult, len(elements)) + for i, element := range elements { + data, err := srcDb.Node(element.hash) if err != nil { - t.Fatalf("failed to retrieve node data for %x: %v", hash, err) + t.Fatalf("failed to retrieve node data for %x: %v", element.hash, err) } - results[i] = SyncResult{hash, data} + results[i] = NodeSyncResult{element.path, data} } for _, result := range results { - if err := sched.Process(result); err != nil { + if err := sched.ProcessNode(result); err != nil { t.Fatalf("failed to process result %v", err) } } @@ -457,9 +559,16 @@ func TestSyncOrdering(t *testing.T) { } batch.Write() - nodes, paths, _ = sched.Missing(1) - queue = append(queue[:0], nodes...) - reqs = append(reqs, paths...) + paths, nodes, _ = sched.Missing(1) + elements = elements[:0] + for i := 0; i < len(paths); i++ { + elements = append(elements, trieElement{ + path: paths[i], + hash: nodes[i], + syncPath: NewSyncPath([]byte(paths[i])), + }) + reqs = append(reqs, NewSyncPath([]byte(paths[i]))) + } } // Cross check that the two tries are in sync checkTrieContents(t, triedb, srcTrie.Hash().Bytes(), srcData) diff --git a/trie/trie.go b/trie/trie.go index fe7d6dc17e79f..bec6a1cc78913 100644 --- a/trie/trie.go +++ b/trie/trie.go @@ -21,14 +21,10 @@ import ( "bytes" "errors" "fmt" - "sync" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/rlp" ) var ( @@ -42,35 +38,40 @@ var ( // LeafCallback is a callback type invoked when a trie operation reaches a leaf // node. // -// The paths is a path tuple identifying a particular trie node either in a single -// trie (account) or a layered trie (account -> storage). Each path in the tuple +// The keys is a path tuple identifying a particular trie node either in a single +// trie (account) or a layered trie (account -> storage). Each key in the tuple // is in the raw format(32 bytes). // -// The hexpath is a composite hexary path identifying the trie node. All the key +// The path is a composite hexary path identifying the trie node. All the key // bytes are converted to the hexary nibbles and composited with the parent path // if the trie node is in a layered trie. // // It's used by state sync and commit to allow handling external references // between account and storage tries. And also it's used in the state healing // for extracting the raw states(leaf nodes) with corresponding paths. -type LeafCallback func(paths [][]byte, hexpath []byte, leaf []byte, parent common.Hash) error +type LeafCallback func(keys [][]byte, path []byte, leaf []byte, parent common.Hash, parentPath []byte) error -// Trie is a Merkle Patricia Trie. -// The zero value is an empty trie with no database. -// Use New to create a trie that sits on top of a database. +// Trie is a Merkle Patricia Trie. Use New to create a trie that sits on +// top of a database. Whenever trie performs a commit operation, the generated +// nodes will be gathered and returned in a set. Once the trie is committed, +// it's not usable anymore. Callers have to re-create the trie with new root +// based on the updated trie database. // // Trie is not safe for concurrent use. type Trie struct { - db *Database - root node + root node + owner common.Hash // Keep track of the number leaves which have been inserted since the last // hashing operation. This number will not directly map to the number of - // actually unhashed nodes + // actually unhashed nodes. unhashed int - // tracer is the state diff tracer can be used to track newly added/deleted - // trie node. It will be reset after each commit operation. + // reader is the handler trie can retrieve nodes from. + reader *trieReader + + // tracer is the tool to track the trie changes. + // It will be reset after each commit operation. tracer *tracer } @@ -82,29 +83,32 @@ func (t *Trie) newFlag() nodeFlag { // Copy returns a copy of Trie. func (t *Trie) Copy() *Trie { return &Trie{ - db: t.db, root: t.root, + owner: t.owner, unhashed: t.unhashed, + reader: t.reader, tracer: t.tracer.copy(), } } -// New creates a trie with an existing root node from db. -// -// If root is the zero hash or the sha3 hash of an empty string, the -// trie is initially empty and does not require a database. Otherwise, -// New will panic if db is nil and returns a MissingNodeError if root does -// not exist in the database. Accessing the trie loads nodes from db on demand. -func New(root common.Hash, db *Database) (*Trie, error) { - if db == nil { - panic("trie.New called without a database") +// New creates the trie instance with provided trie id and the read-only +// database. The state specified by trie id must be available, otherwise +// an error will be returned. The trie root specified by trie id can be +// zero hash or the sha3 hash of an empty string, then trie is initially +// empty, otherwise, the root node must be present in database or returns +// a MissingNodeError if not. +func New(id *ID, db NodeReader) (*Trie, error) { + reader, err := newTrieReader(id.StateRoot, id.Owner, db) + if err != nil { + return nil, err } trie := &Trie{ - db: db, + owner: id.Owner, + reader: reader, //tracer: newTracer(), } - if root != (common.Hash{}) && root != emptyRoot { - rootnode, err := trie.resolveHash(root[:], nil) + if id.Root != (common.Hash{}) && id.Root != emptyRoot { + rootnode, err := trie.resolveAndTrack(id.Root[:], nil) if err != nil { return nil, err } @@ -113,14 +117,10 @@ func New(root common.Hash, db *Database) (*Trie, error) { return trie, nil } -// newWithRootNode initializes the trie with the given root node. -// It's only used by range prover. -func newWithRootNode(root node) *Trie { - return &Trie{ - root: root, - //tracer: newTracer(), - db: NewDatabase(rawdb.NewMemoryDatabase()), - } +// NewEmpty is a shortcut to create empty tree. It's mostly used in tests. +func NewEmpty(db *Database) *Trie { + tr, _ := New(TrieID(common.Hash{}), db) + return tr } // NodeIterator returns an iterator that returns nodes of the trie. Iteration starts at @@ -134,7 +134,7 @@ func (t *Trie) NodeIterator(start []byte) NodeIterator { func (t *Trie) Get(key []byte) []byte { res, err := t.TryGet(key) if err != nil { - log.Error(fmt.Sprintf("Unhandled trie error: %v", err)) + log.Error("Unhandled trie error in Trie.Get", "err", err) } return res } @@ -175,7 +175,7 @@ func (t *Trie) tryGet(origNode node, key []byte, pos int) (value []byte, newnode } return value, n, didResolve, err case hashNode: - child, err := t.resolveHash(n, key[:pos]) + child, err := t.resolveAndTrack(n, key[:pos]) if err != nil { return nil, n, true, err } @@ -221,7 +221,7 @@ func (t *Trie) tryGetNode(origNode node, path []byte, pos int) (item []byte, new if hash == nil { return nil, origNode, 0, errors.New("non-consensus node") } - blob, err := t.db.Node(common.BytesToHash(hash)) + blob, err := t.reader.nodeBlob(path, common.BytesToHash(hash)) return blob, origNode, 1, err } // Path still needs to be traversed, descend into children @@ -251,7 +251,7 @@ func (t *Trie) tryGetNode(origNode node, path []byte, pos int) (item []byte, new return item, n, resolved, err case hashNode: - child, err := t.resolveHash(n, path[:pos]) + child, err := t.resolveAndTrack(n, path[:pos]) if err != nil { return nil, n, 1, err } @@ -271,16 +271,8 @@ func (t *Trie) tryGetNode(origNode node, path []byte, pos int) (item []byte, new // stored in the trie. func (t *Trie) Update(key, value []byte) { if err := t.TryUpdate(key, value); err != nil { - log.Error(fmt.Sprintf("Unhandled trie error: %v", err)) - } -} - -func (t *Trie) TryUpdateAccount(key []byte, acc *types.StateAccount) error { - data, err := rlp.EncodeToBytes(acc) - if err != nil { - return fmt.Errorf("can't encode object at %x: %w", key[:], err) + log.Error("Unhandled trie error in Trie.Update", "err", err) } - return t.TryUpdate(key, data) } // TryUpdate associates key with value in the trie. Subsequent calls to @@ -292,6 +284,12 @@ func (t *Trie) TryUpdateAccount(key []byte, acc *types.StateAccount) error { // // If a node was not found in the database, a MissingNodeError is returned. func (t *Trie) TryUpdate(key, value []byte) error { + return t.tryUpdate(key, value) +} + +// tryUpdate expects an RLP-encoded value and performs the core function +// for TryUpdate and TryUpdateAccount. +func (t *Trie) tryUpdate(key, value []byte) error { t.unhashed++ k := keybytesToHex(key) if len(value) != 0 { @@ -374,7 +372,7 @@ func (t *Trie) insert(n node, prefix, key []byte, value node) (bool, node, error // We've hit a part of the trie that isn't loaded yet. Load // the node and insert into it. This leaves all child nodes on // the path to the value in the trie. - rn, err := t.resolveHash(n, prefix) + rn, err := t.resolveAndTrack(n, prefix) if err != nil { return false, nil, err } @@ -392,7 +390,7 @@ func (t *Trie) insert(n node, prefix, key []byte, value node) (bool, node, error // Delete removes any existing value for key from the trie. func (t *Trie) Delete(key []byte) { if err := t.TryDelete(key); err != nil { - log.Error(fmt.Sprintf("Unhandled trie error: %v", err)) + log.Error("Unhandled trie error in Trie.Delete", "err", err) } } @@ -497,7 +495,7 @@ func (t *Trie) delete(n node, prefix, key []byte) (bool, node, error) { // shortNode{..., shortNode{...}}. Since the entry // might not be loaded yet, resolve it just for this // check. - cnode, err := t.resolve(n.Children[pos], prefix) + cnode, err := t.resolve(n.Children[pos], append(prefix, byte(pos))) if err != nil { return false, nil, err } @@ -528,7 +526,7 @@ func (t *Trie) delete(n node, prefix, key []byte) (bool, node, error) { // We've hit a part of the trie that isn't loaded yet. Load // the node and delete from it. This leaves all child nodes on // the path to the value in the trie. - rn, err := t.resolveHash(n, prefix) + rn, err := t.resolveAndTrack(n, prefix) if err != nil { return false, nil, err } @@ -552,26 +550,22 @@ func concat(s1 []byte, s2 ...byte) []byte { func (t *Trie) resolve(n node, prefix []byte) (node, error) { if n, ok := n.(hashNode); ok { - return t.resolveHash(n, prefix) + return t.resolveAndTrack(n, prefix) } return n, nil } -func (t *Trie) resolveHash(n hashNode, prefix []byte) (node, error) { - hash := common.BytesToHash(n) - if node := t.db.node(hash); node != nil { - return node, nil - } - return nil, &MissingNodeError{NodeHash: hash, Path: prefix} -} - -func (t *Trie) resolveBlob(n hashNode, prefix []byte) ([]byte, error) { - hash := common.BytesToHash(n) - blob, _ := t.db.Node(hash) - if len(blob) != 0 { - return blob, nil +// resolveAndTrack loads node from the underlying store with the given node hash +// and path prefix and also tracks the loaded node blob in tracer treated as the +// node's original value. The rlp-encoded blob is preferred to be loaded from +// database because it's easy to decode node while complex to encode node to blob. +func (t *Trie) resolveAndTrack(n hashNode, prefix []byte) (node, error) { + blob, err := t.reader.nodeBlob(prefix, common.BytesToHash(n)) + if err != nil { + return nil, err } - return nil, &MissingNodeError{NodeHash: hash, Path: prefix} + t.tracer.onRead(prefix, blob) + return mustDecodeNode(n, blob), nil } // Hash returns the root hash of the trie. It does not write to the @@ -582,53 +576,37 @@ func (t *Trie) Hash() common.Hash { return common.BytesToHash(hash.(hashNode)) } -// Commit writes all nodes to the trie's memory database, tracking the internal -// and external (for account tries) references. -func (t *Trie) Commit(onleaf LeafCallback) (common.Hash, int, error) { - if t.db == nil { - panic("commit called on trie with nil database") - } +// Commit collects all dirty nodes in the trie and replaces them with the +// corresponding node hash. All collected nodes (including dirty leaves if +// collectLeaf is true) will be encapsulated into a nodeset for return. +// The returned nodeset can be nil if the trie is clean (nothing to commit). +// Once the trie is committed, it's not usable anymore. A new trie must +// be created with new root and updated trie database for following usage +func (t *Trie) Commit(collectLeaf bool) (common.Hash, *NodeSet, error) { defer t.tracer.reset() if t.root == nil { - return emptyRoot, 0, nil + return emptyRoot, nil, nil } // Derive the hash for all dirty nodes first. We hold the assumption // in the following procedure that all nodes are hashed. rootHash := t.Hash() - h := newCommitter() - defer returnCommitterToPool(h) - - // Do a quick check if we really need to commit, before we spin - // up goroutines. This can happen e.g. if we load a trie for reading storage - // values, but don't write to it. - if _, dirty := t.root.cache(); !dirty { - return rootHash, 0, nil - } - var wg sync.WaitGroup - if onleaf != nil { - h.onleaf = onleaf - h.leafCh = make(chan *leaf, leafChanSize) - wg.Add(1) - go func() { - defer wg.Done() - h.commitLoop(t.db) - }() - } - newRoot, committed, err := h.Commit(t.root, t.db) - if onleaf != nil { - // The leafch is created in newCommitter if there was an onleaf callback - // provided. The commitLoop only _reads_ from it, and the commit - // operation was the sole writer. Therefore, it's safe to close this - // channel here. - close(h.leafCh) - wg.Wait() + + // Do a quick check if we really need to commit. This can happen e.g. + // if we load a trie for reading storage values, but don't write to it. + if hashedNode, dirty := t.root.cache(); !dirty { + // Replace the root node with the origin hash in order to + // ensure all resolved nodes are dropped after the commit. + t.root = hashedNode + return rootHash, nil, nil } + h := newCommitter(t.owner, t.tracer, collectLeaf) + newRoot, nodes, err := h.Commit(t.root) if err != nil { - return common.Hash{}, 0, err + return common.Hash{}, nil, err } t.root = newRoot - return rootHash, committed, nil + return rootHash, nodes, nil } // hashRoot calculates the root hash of the given trie @@ -647,6 +625,7 @@ func (t *Trie) hashRoot() (node, node, error) { // Reset drops the referenced root node and cleans all internal state. func (t *Trie) Reset() { t.root = nil + t.owner = common.Hash{} t.unhashed = 0 t.tracer.reset() } diff --git a/trie/trie_id.go b/trie/trie_id.go new file mode 100644 index 0000000000000..8ab490ca3b1cc --- /dev/null +++ b/trie/trie_id.go @@ -0,0 +1,55 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see + +package trie + +import "github.com/ethereum/go-ethereum/common" + +// ID is the identifier for uniquely identifying a trie. +type ID struct { + StateRoot common.Hash // The root of the corresponding state(block.root) + Owner common.Hash // The contract address hash which the trie belongs to + Root common.Hash // The root hash of trie +} + +// StateTrieID constructs an identifier for state trie with the provided state root. +func StateTrieID(root common.Hash) *ID { + return &ID{ + StateRoot: root, + Owner: common.Hash{}, + Root: root, + } +} + +// StorageTrieID constructs an identifier for storage trie which belongs to a certain +// state and contract specified by the stateRoot and owner. +func StorageTrieID(stateRoot common.Hash, owner common.Hash, root common.Hash) *ID { + return &ID{ + StateRoot: stateRoot, + Owner: owner, + Root: root, + } +} + +// TrieID constructs an identifier for a standard trie(not a second-layer trie) +// with provided root. It's mostly used in tests and some other tries like CHT trie. +func TrieID(root common.Hash) *ID { + return &ID{ + StateRoot: root, + Owner: common.Hash{}, + Root: root, + } +} diff --git a/trie/trie_reader.go b/trie/trie_reader.go new file mode 100644 index 0000000000000..14186159b71bf --- /dev/null +++ b/trie/trie_reader.go @@ -0,0 +1,106 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package trie + +import ( + "fmt" + + "github.com/ethereum/go-ethereum/common" +) + +// Reader wraps the Node and NodeBlob method of a backing trie store. +type Reader interface { + // Node retrieves the trie node with the provided trie identifier, hexary + // node path and the corresponding node hash. + // No error will be returned if the node is not found. + Node(owner common.Hash, path []byte, hash common.Hash) (node, error) + + // NodeBlob retrieves the RLP-encoded trie node blob with the provided trie + // identifier, hexary node path and the corresponding node hash. + // No error will be returned if the node is not found. + NodeBlob(owner common.Hash, path []byte, hash common.Hash) ([]byte, error) +} + +// NodeReader wraps all the necessary functions for accessing trie node. +type NodeReader interface { + // GetReader returns a reader for accessing all trie nodes with provided + // state root. Nil is returned in case the state is not available. + GetReader(root common.Hash) Reader +} + +// trieReader is a wrapper of the underlying node reader. It's not safe +// for concurrent usage. +type trieReader struct { + owner common.Hash + reader Reader + banned map[string]struct{} // Marker to prevent node from being accessed, for tests +} + +// newTrieReader initializes the trie reader with the given node reader. +func newTrieReader(stateRoot, owner common.Hash, db NodeReader) (*trieReader, error) { + reader := db.GetReader(stateRoot) + if reader == nil { + return nil, fmt.Errorf("state not found #%x", stateRoot) + } + return &trieReader{owner: owner, reader: reader}, nil +} + +// newEmptyReader initializes the pure in-memory reader. All read operations +// should be forbidden and returns the MissingNodeError. +func newEmptyReader() *trieReader { + return &trieReader{} +} + +// node retrieves the trie node with the provided trie node information. +// An MissingNodeError will be returned in case the node is not found or +// any error is encountered. +func (r *trieReader) node(path []byte, hash common.Hash) (node, error) { + // Perform the logics in tests for preventing trie node access. + if r.banned != nil { + if _, ok := r.banned[string(path)]; ok { + return nil, &MissingNodeError{Owner: r.owner, NodeHash: hash, Path: path} + } + } + if r.reader == nil { + return nil, &MissingNodeError{Owner: r.owner, NodeHash: hash, Path: path} + } + node, err := r.reader.Node(r.owner, path, hash) + if err != nil || node == nil { + return nil, &MissingNodeError{Owner: r.owner, NodeHash: hash, Path: path, err: err} + } + return node, nil +} + +// node retrieves the rlp-encoded trie node with the provided trie node +// information. An MissingNodeError will be returned in case the node is +// not found or any error is encountered. +func (r *trieReader) nodeBlob(path []byte, hash common.Hash) ([]byte, error) { + // Perform the logics in tests for preventing trie node access. + if r.banned != nil { + if _, ok := r.banned[string(path)]; ok { + return nil, &MissingNodeError{Owner: r.owner, NodeHash: hash, Path: path} + } + } + if r.reader == nil { + return nil, &MissingNodeError{Owner: r.owner, NodeHash: hash, Path: path} + } + blob, err := r.reader.NodeBlob(r.owner, path, hash) + if err != nil || len(blob) == 0 { + return nil, &MissingNodeError{Owner: r.owner, NodeHash: hash, Path: path, err: err} + } + return blob, nil +} diff --git a/trie/trie_test.go b/trie/trie_test.go index f994e31af40e1..d2a599ffdd64f 100644 --- a/trie/trie_test.go +++ b/trie/trie_test.go @@ -24,7 +24,6 @@ import ( "hash" "math/big" "math/rand" - "os" "reflect" "testing" "testing/quick" @@ -35,7 +34,6 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/ethdb/leveldb" "github.com/ethereum/go-ethereum/ethdb/memorydb" "github.com/ethereum/go-ethereum/rlp" "golang.org/x/crypto/sha3" @@ -46,14 +44,8 @@ func init() { spew.Config.DisableMethods = false } -// Used for testing -func newEmpty() *Trie { - trie, _ := New(common.Hash{}, NewDatabase(memorydb.New())) - return trie -} - func TestEmptyTrie(t *testing.T) { - trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) res := trie.Hash() exp := emptyRoot if res != exp { @@ -62,7 +54,7 @@ func TestEmptyTrie(t *testing.T) { } func TestNull(t *testing.T) { - trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) key := make([]byte, 32) value := []byte("test") trie.Update(key, value) @@ -72,7 +64,8 @@ func TestNull(t *testing.T) { } func TestMissingRoot(t *testing.T) { - trie, err := New(common.HexToHash("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33"), NewDatabase(memorydb.New())) + root := common.HexToHash("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33") + trie, err := New(TrieID(root), NewDatabase(memorydb.New())) if trie != nil { t.Error("New returned non-nil trie for invalid root") } @@ -88,35 +81,36 @@ func testMissingNode(t *testing.T, memonly bool) { diskdb := memorydb.New() triedb := NewDatabase(diskdb) - trie, _ := New(common.Hash{}, triedb) + trie := NewEmpty(triedb) updateString(trie, "120000", "qwerqwerqwerqwerqwerqwerqwerqwer") updateString(trie, "123456", "asdfasdfasdfasdfasdfasdfasdfasdf") - root, _, _ := trie.Commit(nil) + root, nodes, _ := trie.Commit(false) + triedb.Update(NewWithNodeSet(nodes)) if !memonly { triedb.Commit(root, true, nil) } - trie, _ = New(root, triedb) + trie, _ = New(TrieID(root), triedb) _, err := trie.TryGet([]byte("120000")) if err != nil { t.Errorf("Unexpected error: %v", err) } - trie, _ = New(root, triedb) + trie, _ = New(TrieID(root), triedb) _, err = trie.TryGet([]byte("120099")) if err != nil { t.Errorf("Unexpected error: %v", err) } - trie, _ = New(root, triedb) + trie, _ = New(TrieID(root), triedb) _, err = trie.TryGet([]byte("123456")) if err != nil { t.Errorf("Unexpected error: %v", err) } - trie, _ = New(root, triedb) + trie, _ = New(TrieID(root), triedb) err = trie.TryUpdate([]byte("120099"), []byte("zxcvzxcvzxcvzxcvzxcvzxcvzxcvzxcv")) if err != nil { t.Errorf("Unexpected error: %v", err) } - trie, _ = New(root, triedb) + trie, _ = New(TrieID(root), triedb) err = trie.TryDelete([]byte("123456")) if err != nil { t.Errorf("Unexpected error: %v", err) @@ -129,27 +123,27 @@ func testMissingNode(t *testing.T, memonly bool) { diskdb.Delete(hash[:]) } - trie, _ = New(root, triedb) + trie, _ = New(TrieID(root), triedb) _, err = trie.TryGet([]byte("120000")) if _, ok := err.(*MissingNodeError); !ok { t.Errorf("Wrong error: %v", err) } - trie, _ = New(root, triedb) + trie, _ = New(TrieID(root), triedb) _, err = trie.TryGet([]byte("120099")) if _, ok := err.(*MissingNodeError); !ok { t.Errorf("Wrong error: %v", err) } - trie, _ = New(root, triedb) + trie, _ = New(TrieID(root), triedb) _, err = trie.TryGet([]byte("123456")) if err != nil { t.Errorf("Unexpected error: %v", err) } - trie, _ = New(root, triedb) + trie, _ = New(TrieID(root), triedb) err = trie.TryUpdate([]byte("120099"), []byte("zxcv")) if _, ok := err.(*MissingNodeError); !ok { t.Errorf("Wrong error: %v", err) } - trie, _ = New(root, triedb) + trie, _ = New(TrieID(root), triedb) err = trie.TryDelete([]byte("123456")) if _, ok := err.(*MissingNodeError); !ok { t.Errorf("Wrong error: %v", err) @@ -157,7 +151,7 @@ func testMissingNode(t *testing.T, memonly bool) { } func TestInsert(t *testing.T) { - trie := newEmpty() + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) updateString(trie, "doe", "reindeer") updateString(trie, "dog", "puppy") @@ -169,11 +163,11 @@ func TestInsert(t *testing.T) { t.Errorf("case 1: exp %x got %x", exp, root) } - trie = newEmpty() + trie = NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) updateString(trie, "A", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") exp = common.HexToHash("d23786fb4a010da3ce639d66d5e904a11dbc02746d1ce25029e53290cabf28ab") - root, _, err := trie.Commit(nil) + root, _, err := trie.Commit(false) if err != nil { t.Fatalf("commit error: %v", err) } @@ -183,7 +177,8 @@ func TestInsert(t *testing.T) { } func TestGet(t *testing.T) { - trie := newEmpty() + db := NewDatabase(rawdb.NewMemoryDatabase()) + trie := NewEmpty(db) updateString(trie, "doe", "reindeer") updateString(trie, "dog", "puppy") updateString(trie, "dogglesworth", "cat") @@ -193,21 +188,21 @@ func TestGet(t *testing.T) { if !bytes.Equal(res, []byte("puppy")) { t.Errorf("expected puppy got %x", res) } - unknown := getString(trie, "unknown") if unknown != nil { t.Errorf("expected nil got %x", unknown) } - if i == 1 { return } - trie.Commit(nil) + root, nodes, _ := trie.Commit(false) + db.Update(NewWithNodeSet(nodes)) + trie, _ = New(TrieID(root), db) } } func TestDelete(t *testing.T) { - trie := newEmpty() + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) vals := []struct{ k, v string }{ {"do", "verb"}, {"ether", "wookiedoo"}, @@ -234,7 +229,7 @@ func TestDelete(t *testing.T) { } func TestEmptyValues(t *testing.T) { - trie := newEmpty() + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) vals := []struct{ k, v string }{ {"do", "verb"}, @@ -258,7 +253,8 @@ func TestEmptyValues(t *testing.T) { } func TestReplication(t *testing.T) { - trie := newEmpty() + triedb := NewDatabase(rawdb.NewMemoryDatabase()) + trie := NewEmpty(triedb) vals := []struct{ k, v string }{ {"do", "verb"}, {"ether", "wookiedoo"}, @@ -271,13 +267,14 @@ func TestReplication(t *testing.T) { for _, val := range vals { updateString(trie, val.k, val.v) } - exp, _, err := trie.Commit(nil) + exp, nodes, err := trie.Commit(false) if err != nil { t.Fatalf("commit error: %v", err) } + triedb.Update(NewWithNodeSet(nodes)) // create a new trie on top of the database and check that lookups work. - trie2, err := New(exp, trie.db) + trie2, err := New(TrieID(exp), triedb) if err != nil { t.Fatalf("can't recreate trie at %x: %v", exp, err) } @@ -286,7 +283,7 @@ func TestReplication(t *testing.T) { t.Errorf("trie2 doesn't have %q => %q", kv.k, kv.v) } } - hash, _, err := trie2.Commit(nil) + hash, nodes, err := trie2.Commit(false) if err != nil { t.Fatalf("commit error: %v", err) } @@ -294,6 +291,14 @@ func TestReplication(t *testing.T) { t.Errorf("root failure. expected %x got %x", exp, hash) } + // recreate the trie after commit + if nodes != nil { + triedb.Update(NewWithNodeSet(nodes)) + } + trie2, err = New(TrieID(hash), triedb) + if err != nil { + t.Fatalf("can't recreate trie at %x: %v", exp, err) + } // perform some insertions on the new trie. vals2 := []struct{ k, v string }{ {"do", "verb"}, @@ -315,7 +320,7 @@ func TestReplication(t *testing.T) { } func TestLargeValue(t *testing.T) { - trie := newEmpty() + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) trie.Update([]byte("key1"), []byte{99, 99, 99, 99}) trie.Update([]byte("key2"), bytes.Repeat([]byte{1}, 32)) trie.Hash() @@ -352,7 +357,6 @@ func TestRandomCases(t *testing.T) { {op: 1, key: common.Hex2Bytes("fd"), value: common.Hex2Bytes("")}, // step 25 } runRandTest(rt) - } // randTest performs random trie operations. @@ -370,11 +374,11 @@ const ( opUpdate = iota opDelete opGet - opCommit opHash - opReset + opCommit opItercheckhash opNodeDiff + opProve opMax // boundary value, not an actual op ) @@ -400,7 +404,7 @@ func (randTest) Generate(r *rand.Rand, size int) reflect.Value { step.key = genKey() step.value = make([]byte, 8) binary.BigEndian.PutUint64(step.value, uint64(i)) - case opGet, opDelete: + case opGet, opDelete, opProve: step.key = genKey() } steps = append(steps, step) @@ -410,10 +414,10 @@ func (randTest) Generate(r *rand.Rand, size int) reflect.Value { func runRandTest(rt randTest) bool { var ( - triedb = NewDatabase(memorydb.New()) - tr, _ = New(common.Hash{}, triedb) - values = make(map[string]string) // tracks content of the trie - origTrie, _ = New(common.Hash{}, triedb) + triedb = NewDatabase(memorydb.New()) + tr = NewEmpty(triedb) + values = make(map[string]string) // tracks content of the trie + origTrie = NewEmpty(triedb) ) tr.tracer = newTracer() @@ -432,30 +436,66 @@ func runRandTest(rt randTest) bool { v := tr.Get(step.key) want := values[string(step.key)] if string(v) != want { - rt[i].err = fmt.Errorf("mismatch for key 0x%x, got 0x%x want 0x%x", step.key, v, want) + rt[i].err = fmt.Errorf("mismatch for key %#x, got %#x want %#x", step.key, v, want) + } + case opProve: + hash := tr.Hash() + if hash == emptyRoot { + continue + } + proofDb := rawdb.NewMemoryDatabase() + err := tr.Prove(step.key, 0, proofDb) + if err != nil { + rt[i].err = fmt.Errorf("failed for proving key %#x, %v", step.key, err) + } + _, err = VerifyProof(hash, step.key, proofDb) + if err != nil { + rt[i].err = fmt.Errorf("failed for verifying key %#x, %v", step.key, err) } - case opCommit: - _, _, rt[i].err = tr.Commit(nil) - origTrie = tr.Copy() case opHash: tr.Hash() - case opReset: - hash, _, err := tr.Commit(nil) + case opCommit: + root, nodes, err := tr.Commit(true) if err != nil { rt[i].err = err return false } - newtr, err := New(hash, triedb) + // Validity the returned nodeset + if nodes != nil { + for path, node := range nodes.updates.nodes { + blob, _, _ := origTrie.TryGetNode(hexToCompact([]byte(path))) + got := node.prev + if !bytes.Equal(blob, got) { + rt[i].err = fmt.Errorf("prevalue mismatch for 0x%x, got 0x%x want 0x%x", path, got, blob) + panic(rt[i].err) + } + } + for path, prev := range nodes.deletes { + blob, _, _ := origTrie.TryGetNode(hexToCompact([]byte(path))) + if !bytes.Equal(blob, prev) { + rt[i].err = fmt.Errorf("prevalue mismatch for 0x%x, got 0x%x want 0x%x", path, prev, blob) + return false + } + } + } + if nodes != nil { + triedb.Update(NewWithNodeSet(nodes)) + } + newtr, err := New(TrieID(root), triedb) if err != nil { rt[i].err = err return false } tr = newtr + + // Enable node tracing. Resolve the root node again explicitly + // since it's not captured at the beginning. tr.tracer = newTracer() + tr.resolveAndTrack(root.Bytes(), nil) origTrie = tr.Copy() case opItercheckhash: - checktr, _ := New(common.Hash{}, triedb) + checktr := NewEmpty(triedb) it := NewIterator(tr.NodeIterator(nil)) for it.Next() { checktr.Update(it.Key, it.Value) @@ -534,44 +574,31 @@ func TestRandom(t *testing.T) { } } -func BenchmarkGet(b *testing.B) { benchGet(b, false) } -func BenchmarkGetDB(b *testing.B) { benchGet(b, true) } +func BenchmarkGet(b *testing.B) { benchGet(b) } func BenchmarkUpdateBE(b *testing.B) { benchUpdate(b, binary.BigEndian) } func BenchmarkUpdateLE(b *testing.B) { benchUpdate(b, binary.LittleEndian) } const benchElemCount = 20000 -func benchGet(b *testing.B, commit bool) { - trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase())) - if commit { - tmpdb := tempDB(b) - trie, _ = New(common.Hash{}, tmpdb) - } +func benchGet(b *testing.B) { + triedb := NewDatabase(rawdb.NewMemoryDatabase()) + trie := NewEmpty(triedb) k := make([]byte, 32) for i := 0; i < benchElemCount; i++ { binary.LittleEndian.PutUint64(k, uint64(i)) trie.Update(k, k) } binary.LittleEndian.PutUint64(k, benchElemCount/2) - if commit { - trie.Commit(nil) - } b.ResetTimer() for i := 0; i < b.N; i++ { trie.Get(k) } b.StopTimer() - - if commit { - ldb := trie.db.diskdb.(*leveldb.Database) - ldb.Close() - os.RemoveAll(ldb.Path()) - } } func benchUpdate(b *testing.B, e binary.ByteOrder) *Trie { - trie := newEmpty() + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) k := make([]byte, 32) b.ReportAllocs() for i := 0; i < b.N; i++ { @@ -601,7 +628,7 @@ func BenchmarkHash(b *testing.B) { // entries, then adding N more. addresses, accounts := makeAccounts(2 * b.N) // Insert the accounts into the trie and hash it - trie := newEmpty() + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) i := 0 for ; i < len(addresses)/2; i++ { trie.Update(crypto.Keccak256(addresses[i][:]), accounts[i]) @@ -622,22 +649,17 @@ func BenchmarkHash(b *testing.B) { // insert into the trie before measuring the hashing. func BenchmarkCommitAfterHash(b *testing.B) { b.Run("no-onleaf", func(b *testing.B) { - benchmarkCommitAfterHash(b, nil) + benchmarkCommitAfterHash(b, false) }) - var a types.StateAccount - onleaf := func(paths [][]byte, hexpath []byte, leaf []byte, parent common.Hash) error { - rlp.DecodeBytes(leaf, &a) - return nil - } b.Run("with-onleaf", func(b *testing.B) { - benchmarkCommitAfterHash(b, onleaf) + benchmarkCommitAfterHash(b, true) }) } -func benchmarkCommitAfterHash(b *testing.B, onleaf LeafCallback) { +func benchmarkCommitAfterHash(b *testing.B, collectLeaf bool) { // Make the random benchmark deterministic addresses, accounts := makeAccounts(b.N) - trie := newEmpty() + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) for i := 0; i < len(addresses); i++ { trie.Update(crypto.Keccak256(addresses[i][:]), accounts[i]) } @@ -645,13 +667,13 @@ func benchmarkCommitAfterHash(b *testing.B, onleaf LeafCallback) { trie.Hash() b.ResetTimer() b.ReportAllocs() - trie.Commit(onleaf) + trie.Commit(collectLeaf) } func TestTinyTrie(t *testing.T) { // Create a realistic account trie to hash _, accounts := makeAccounts(5) - trie := newEmpty() + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) trie.Update(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000001337"), accounts[3]) if exp, root := common.HexToHash("8c6a85a4d9fda98feff88450299e574e5378e32391f75a055d470ac0653f1005"), trie.Hash(); exp != root { t.Errorf("1: got %x, exp %x", root, exp) @@ -664,7 +686,7 @@ func TestTinyTrie(t *testing.T) { if exp, root := common.HexToHash("0608c1d1dc3905fa22204c7a0e43644831c3b6d3def0f274be623a948197e64a"), trie.Hash(); exp != root { t.Errorf("3: got %x, exp %x", root, exp) } - checktr, _ := New(common.Hash{}, trie.db) + checktr := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) it := NewIterator(trie.NodeIterator(nil)) for it.Next() { checktr.Update(it.Key, it.Value) @@ -677,19 +699,19 @@ func TestTinyTrie(t *testing.T) { func TestCommitAfterHash(t *testing.T) { // Create a realistic account trie to hash addresses, accounts := makeAccounts(1000) - trie := newEmpty() + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) for i := 0; i < len(addresses); i++ { trie.Update(crypto.Keccak256(addresses[i][:]), accounts[i]) } // Insert the accounts into the trie and hash it trie.Hash() - trie.Commit(nil) + trie.Commit(false) root := trie.Hash() exp := common.HexToHash("72f9d3f3fe1e1dd7b8936442e7642aef76371472d94319900790053c493f3fe6") if exp != root { t.Errorf("got %x, exp %x", root, exp) } - root, _, _ = trie.Commit(nil) + root, _, _ = trie.Commit(false) if exp != root { t.Errorf("got %x, exp %x", root, exp) } @@ -790,7 +812,7 @@ func TestCommitSequence(t *testing.T) { // This spongeDb is used to check the sequence of disk-db-writes s := &spongeDb{sponge: sha3.NewLegacyKeccak256()} db := NewDatabase(s) - trie, _ := New(common.Hash{}, db) + trie := NewEmpty(db) // Another sponge is used to check the callback-sequence callbackSponge := sha3.NewLegacyKeccak256() // Fill the trie with elements @@ -798,7 +820,8 @@ func TestCommitSequence(t *testing.T) { trie.Update(crypto.Keccak256(addresses[i][:]), accounts[i]) } // Flush trie -> database - root, _, _ := trie.Commit(nil) + root, nodes, _ := trie.Commit(false) + db.Update(NewWithNodeSet(nodes)) // Flush memdb -> disk (sponge) db.Commit(root, false, func(c common.Hash) { // And spongify the callback-order @@ -832,7 +855,7 @@ func TestCommitSequenceRandomBlobs(t *testing.T) { // This spongeDb is used to check the sequence of disk-db-writes s := &spongeDb{sponge: sha3.NewLegacyKeccak256()} db := NewDatabase(s) - trie, _ := New(common.Hash{}, db) + trie := NewEmpty(db) // Another sponge is used to check the callback-sequence callbackSponge := sha3.NewLegacyKeccak256() // Fill the trie with elements @@ -850,7 +873,8 @@ func TestCommitSequenceRandomBlobs(t *testing.T) { trie.Update(key, val) } // Flush trie -> database - root, _, _ := trie.Commit(nil) + root, nodes, _ := trie.Commit(false) + db.Update(NewWithNodeSet(nodes)) // Flush memdb -> disk (sponge) db.Commit(root, false, func(c common.Hash) { // And spongify the callback-order @@ -871,12 +895,12 @@ func TestCommitSequenceStackTrie(t *testing.T) { // This spongeDb is used to check the sequence of disk-db-writes s := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "a"} db := NewDatabase(s) - trie, _ := New(common.Hash{}, db) + trie := NewEmpty(db) // Another sponge is used for the stacktrie commits stackTrieSponge := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "b"} stTrie := NewStackTrie(stackTrieSponge) // Fill the trie with elements - for i := 1; i < count; i++ { + for i := 0; i < count; i++ { // For the stack trie, we need to do inserts in proper order key := make([]byte, 32) binary.BigEndian.PutUint64(key, uint64(i)) @@ -892,8 +916,9 @@ func TestCommitSequenceStackTrie(t *testing.T) { stTrie.TryUpdate(key, val) } // Flush trie -> database - root, _, _ := trie.Commit(nil) + root, nodes, _ := trie.Commit(false) // Flush memdb -> disk (sponge) + db.Update(NewWithNodeSet(nodes)) db.Commit(root, false, nil) // And flush stacktrie -> disk stRoot, err := stTrie.Commit() @@ -927,7 +952,7 @@ func TestCommitSequenceStackTrie(t *testing.T) { func TestCommitSequenceSmallRoot(t *testing.T) { s := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "a"} db := NewDatabase(s) - trie, _ := New(common.Hash{}, db) + trie := NewEmpty(db) // Another sponge is used for the stacktrie commits stackTrieSponge := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "b"} stTrie := NewStackTrie(stackTrieSponge) @@ -937,8 +962,9 @@ func TestCommitSequenceSmallRoot(t *testing.T) { trie.TryUpdate(key, []byte{0x1}) stTrie.TryUpdate(key, []byte{0x1}) // Flush trie -> database - root, _, _ := trie.Commit(nil) + root, nodes, _ := trie.Commit(false) // Flush memdb -> disk (sponge) + db.Update(NewWithNodeSet(nodes)) db.Commit(root, false, nil) // And flush stacktrie -> disk stRoot, err := stTrie.Commit() @@ -1000,7 +1026,7 @@ func BenchmarkHashFixedSize(b *testing.B) { func benchmarkHashFixedSize(b *testing.B, addresses [][20]byte, accounts [][]byte) { b.ReportAllocs() - trie := newEmpty() + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) for i := 0; i < len(addresses); i++ { trie.Update(crypto.Keccak256(addresses[i][:]), accounts[i]) } @@ -1051,14 +1077,14 @@ func BenchmarkCommitAfterHashFixedSize(b *testing.B) { func benchmarkCommitAfterHashFixedSize(b *testing.B, addresses [][20]byte, accounts [][]byte) { b.ReportAllocs() - trie := newEmpty() + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) for i := 0; i < len(addresses); i++ { trie.Update(crypto.Keccak256(addresses[i][:]), accounts[i]) } // Insert the accounts into the trie and hash it trie.Hash() b.StartTimer() - trie.Commit(nil) + trie.Commit(false) b.StopTimer() } @@ -1103,26 +1129,19 @@ func BenchmarkDerefRootFixedSize(b *testing.B) { func benchmarkDerefRootFixedSize(b *testing.B, addresses [][20]byte, accounts [][]byte) { b.ReportAllocs() - trie := newEmpty() + triedb := NewDatabase(rawdb.NewMemoryDatabase()) + trie := NewEmpty(triedb) for i := 0; i < len(addresses); i++ { trie.Update(crypto.Keccak256(addresses[i][:]), accounts[i]) } h := trie.Hash() - trie.Commit(nil) + _, nodes, _ := trie.Commit(false) + triedb.Update(NewWithNodeSet(nodes)) b.StartTimer() - trie.db.Dereference(h) + triedb.Dereference(h) b.StopTimer() } -func tempDB(tb testing.TB) *Database { - dir := tb.TempDir() - diskdb, err := leveldb.New(dir, 256, 0, "", false) - if err != nil { - panic(fmt.Sprintf("can't create temporary database: %v", err)) - } - return NewDatabase(diskdb) -} - func getString(trie *Trie, k string) []byte { return trie.Get([]byte(k)) } diff --git a/trie/util_test.go b/trie/util_test.go index fadb0553b5298..e0e3142050354 100644 --- a/trie/util_test.go +++ b/trie/util_test.go @@ -17,6 +17,7 @@ package trie import ( + "bytes" "testing" "github.com/ethereum/go-ethereum/common" @@ -26,7 +27,7 @@ import ( // Tests if the trie diffs are tracked correctly. func TestTrieTracer(t *testing.T) { db := NewDatabase(rawdb.NewMemoryDatabase()) - trie, _ := New(common.Hash{}, db) + trie := NewEmpty(db) trie.tracer = newTracer() // Insert a batch of entries, all the nodes should be marked as inserted @@ -67,8 +68,13 @@ func TestTrieTracer(t *testing.T) { t.Fatalf("Unexpected deleted node tracked %d", len(deleted)) } - // Commit the changes - trie.Commit(nil) + // Commit the changes and re-create with new root + root, nodes, _ := trie.Commit(false) + if err := db.Update(NewWithNodeSet(nodes)); err != nil { + t.Fatal(err) + } + trie, _ = New(TrieID(root), db) + trie.tracer = newTracer() // Delete all the elements, check deletion set for _, val := range vals { @@ -93,8 +99,7 @@ func TestTrieTracer(t *testing.T) { } func TestTrieTracerNoop(t *testing.T) { - db := NewDatabase(rawdb.NewMemoryDatabase()) - trie, _ := New(common.Hash{}, db) + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) trie.tracer = newTracer() // Insert a batch of entries, all the nodes should be marked as inserted @@ -120,3 +125,120 @@ func TestTrieTracerNoop(t *testing.T) { t.Fatalf("Unexpected deleted node tracked %d", len(trie.tracer.deleteList())) } } + +func TestTrieTracePrevValue(t *testing.T) { + db := NewDatabase(rawdb.NewMemoryDatabase()) + trie := NewEmpty(db) + trie.tracer = newTracer() + + paths, blobs := trie.tracer.prevList() + if len(paths) != 0 || len(blobs) != 0 { + t.Fatalf("Nothing should be tracked") + } + // Insert a batch of entries, all the nodes should be marked as inserted + vals := []struct{ k, v string }{ + {"do", "verb"}, + {"ether", "wookiedoo"}, + {"horse", "stallion"}, + {"shaman", "horse"}, + {"doge", "coin"}, + {"dog", "puppy"}, + {"somethingveryoddindeedthis is", "myothernodedata"}, + } + for _, val := range vals { + trie.Update([]byte(val.k), []byte(val.v)) + } + paths, blobs = trie.tracer.prevList() + if len(paths) != 0 || len(blobs) != 0 { + t.Fatalf("Nothing should be tracked") + } + + // Commit the changes and re-create with new root + root, nodes, _ := trie.Commit(false) + if err := db.Update(NewWithNodeSet(nodes)); err != nil { + t.Fatal(err) + } + trie, _ = New(TrieID(root), db) + trie.tracer = newTracer() + trie.resolveAndTrack(root.Bytes(), nil) + + // Load all nodes in trie + for _, val := range vals { + trie.TryGet([]byte(val.k)) + } + + // Ensure all nodes are tracked by tracer with correct prev-values + iter := trie.NodeIterator(nil) + seen := make(map[string][]byte) + for iter.Next(true) { + // Embedded nodes are ignored since they are not present in + // database. + if iter.Hash() == (common.Hash{}) { + continue + } + seen[string(iter.Path())] = common.CopyBytes(iter.NodeBlob()) + } + + paths, blobs = trie.tracer.prevList() + if len(paths) != len(seen) || len(blobs) != len(seen) { + t.Fatalf("Unexpected tracked values") + } + for i, path := range paths { + blob := blobs[i] + prev, ok := seen[string(path)] + if !ok { + t.Fatalf("Missing node %v", path) + } + if !bytes.Equal(blob, prev) { + t.Fatalf("Unexpected value path: %v, want: %v, got: %v", path, prev, blob) + } + } + + // Re-open the trie and iterate the trie, ensure nothing will be tracked. + // Iterator will not link any loaded nodes to trie. + trie, _ = New(TrieID(root), db) + trie.tracer = newTracer() + + iter = trie.NodeIterator(nil) + for iter.Next(true) { + } + paths, blobs = trie.tracer.prevList() + if len(paths) != 0 || len(blobs) != 0 { + t.Fatalf("Nothing should be tracked") + } + + // Re-open the trie and generate proof for entries, ensure nothing will + // be tracked. Prover will not link any loaded nodes to trie. + trie, _ = New(TrieID(root), db) + trie.tracer = newTracer() + for _, val := range vals { + trie.Prove([]byte(val.k), 0, rawdb.NewMemoryDatabase()) + } + paths, blobs = trie.tracer.prevList() + if len(paths) != 0 || len(blobs) != 0 { + t.Fatalf("Nothing should be tracked") + } + + // Delete entries from trie, ensure all previous values are correct. + trie, _ = New(TrieID(root), db) + trie.tracer = newTracer() + trie.resolveAndTrack(root.Bytes(), nil) + + for _, val := range vals { + trie.TryDelete([]byte(val.k)) + } + paths, blobs = trie.tracer.prevList() + if len(paths) != len(seen) || len(blobs) != len(seen) { + t.Fatalf("Unexpected tracked values") + } + for i, path := range paths { + blob := blobs[i] + prev, ok := seen[string(path)] + if !ok { + t.Fatalf("Missing node %v", path) + } + if !bytes.Equal(blob, prev) { + t.Fatalf("Unexpected value path: %v, want: %v, got: %v", path, prev, blob) + } + } +} diff --git a/trie/utils.go b/trie/utils.go index 5f9e3ba58ecc3..d462b31bd205a 100644 --- a/trie/utils.go +++ b/trie/utils.go @@ -29,49 +29,64 @@ package trie // This tool can track all of them no matter the node is embedded in its // parent or not, but valueNode is never tracked. // +// Besides, it's also used for recording the original value of the nodes +// when they are resolved from the disk. The pre-value of the nodes will +// be used to construct reverse-diffs in the future. +// // Note tracer is not thread-safe, callers should be responsible for handling // the concurrency issues by themselves. type tracer struct { insert map[string]struct{} delete map[string]struct{} + origin map[string][]byte } -// newTracer initializes trie node diff tracer. +// newTracer initializes the tracer for capturing trie changes. func newTracer() *tracer { return &tracer{ insert: make(map[string]struct{}), delete: make(map[string]struct{}), + origin: make(map[string][]byte), + } +} + +// onRead tracks the newly loaded trie node and caches the rlp-encoded blob internally. +// Don't change the value outside of function since it's not deep-copied. +func (t *tracer) onRead(path []byte, val []byte) { + // Tracer isn't used right now, remove this check later. + if t == nil { + return } + t.origin[string(path)] = val } -// onInsert tracks the newly inserted trie node. If it's already -// in the deletion set(resurrected node), then just wipe it from -// the deletion set as it's untouched. -func (t *tracer) onInsert(key []byte) { +// onInsert tracks the newly inserted trie node. If it's already in the deletion set +// (resurrected node), then just wipe it from the deletion set as the "untouched". +func (t *tracer) onInsert(path []byte) { // Tracer isn't used right now, remove this check later. if t == nil { return } - if _, present := t.delete[string(key)]; present { - delete(t.delete, string(key)) + if _, present := t.delete[string(path)]; present { + delete(t.delete, string(path)) return } - t.insert[string(key)] = struct{}{} + t.insert[string(path)] = struct{}{} } // onDelete tracks the newly deleted trie node. If it's already // in the addition set, then just wipe it from the addition set // as it's untouched. -func (t *tracer) onDelete(key []byte) { +func (t *tracer) onDelete(path []byte) { // Tracer isn't used right now, remove this check later. if t == nil { return } - if _, present := t.insert[string(key)]; present { - delete(t.insert, string(key)) + if _, present := t.insert[string(path)]; present { + delete(t.insert, string(path)) return } - t.delete[string(key)] = struct{}{} + t.delete[string(path)] = struct{}{} } // insertList returns the tracked inserted trie nodes in list format. @@ -81,8 +96,8 @@ func (t *tracer) insertList() [][]byte { return nil } var ret [][]byte - for key := range t.insert { - ret = append(ret, []byte(key)) + for path := range t.insert { + ret = append(ret, []byte(path)) } return ret } @@ -94,12 +109,38 @@ func (t *tracer) deleteList() [][]byte { return nil } var ret [][]byte - for key := range t.delete { - ret = append(ret, []byte(key)) + for path := range t.delete { + ret = append(ret, []byte(path)) } return ret } +// prevList returns the tracked node blobs in list format. +func (t *tracer) prevList() ([][]byte, [][]byte) { + // Tracer isn't used right now, remove this check later. + if t == nil { + return nil, nil + } + var ( + paths [][]byte + blobs [][]byte + ) + for path, blob := range t.origin { + paths = append(paths, []byte(path)) + blobs = append(blobs, blob) + } + return paths, blobs +} + +// getPrev returns the cached original value of the specified node. +func (t *tracer) getPrev(path []byte) []byte { + // Tracer isn't used right now, remove this check later. + if t == nil { + return nil + } + return t.origin[string(path)] +} + // reset clears the content tracked by tracer. func (t *tracer) reset() { // Tracer isn't used right now, remove this check later. @@ -108,6 +149,7 @@ func (t *tracer) reset() { } t.insert = make(map[string]struct{}) t.delete = make(map[string]struct{}) + t.origin = make(map[string][]byte) } // copy returns a deep copied tracer instance. @@ -119,6 +161,7 @@ func (t *tracer) copy() *tracer { var ( insert = make(map[string]struct{}) delete = make(map[string]struct{}) + origin = make(map[string][]byte) ) for key := range t.insert { insert[key] = struct{}{} @@ -126,8 +169,12 @@ func (t *tracer) copy() *tracer { for key := range t.delete { delete[key] = struct{}{} } + for key, val := range t.origin { + origin[key] = val + } return &tracer{ insert: insert, delete: delete, + origin: origin, } }